From be5a8fec1f9ca315715e015c2b0e03a7b10172c4 Mon Sep 17 00:00:00 2001 From: zzpwelkin Date: Thu, 26 Dec 2013 14:24:54 +0800 Subject: [PATCH 01/85] Update manifest.yml --- metadata/manifest.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/metadata/manifest.yml b/metadata/manifest.yml index d0ba915..75ca44c 100644 --- a/metadata/manifest.yml +++ b/metadata/manifest.yml @@ -3,7 +3,6 @@ Cartridge-Short-Name: CELERY Display-Name: Celery Cartridge Description: "A cartridge for running the celeryd daemon to allow asynchronous tasks in your application." Version: '0.1' -License: "None" Vendor: Custom Cartridges Inc Cartridge-Version: 0.0.1 Cartridge-Vendor: customcarts From a07b99e8a478b237763b54726e8137bd4a362169 Mon Sep 17 00:00:00 2001 From: zzpwelkin Date: Thu, 26 Dec 2013 14:25:19 +0800 Subject: [PATCH 02/85] Update manifest.yml --- metadata/manifest.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metadata/manifest.yml b/metadata/manifest.yml index 75ca44c..10cd14d 100644 --- a/metadata/manifest.yml +++ b/metadata/manifest.yml @@ -6,7 +6,7 @@ Version: '0.1' Vendor: Custom Cartridges Inc Cartridge-Version: 0.0.1 Cartridge-Vendor: customcarts -Source-Url: https://github.com/tresbailey/openshift-celery-cartridge.git +Source-Url: https://github.com/zzpwelkin/openshift-celery-cartridge.git Categories: - service Provides: From 5146c04ad0e76fe2ca3fb1664a13de6942f6ed9f Mon Sep 17 00:00:00 2001 From: zzpwelkin Date: Thu, 26 Dec 2013 14:27:01 +0800 Subject: [PATCH 03/85] Update manifest.yml --- metadata/manifest.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/metadata/manifest.yml b/metadata/manifest.yml index 10cd14d..f8838ac 100644 --- a/metadata/manifest.yml +++ b/metadata/manifest.yml @@ -3,7 +3,6 @@ Cartridge-Short-Name: CELERY Display-Name: Celery Cartridge Description: "A cartridge for running the celeryd daemon to allow asynchronous tasks in your application." Version: '0.1' -Vendor: Custom Cartridges Inc Cartridge-Version: 0.0.1 Cartridge-Vendor: customcarts Source-Url: https://github.com/zzpwelkin/openshift-celery-cartridge.git From ce63dc433472cb4d700da8bf2e8a2bab194ed018 Mon Sep 17 00:00:00 2001 From: zzpwelkin Date: Thu, 26 Dec 2013 14:29:27 +0800 Subject: [PATCH 04/85] Update manifest.yml --- metadata/manifest.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/metadata/manifest.yml b/metadata/manifest.yml index f8838ac..463ab72 100644 --- a/metadata/manifest.yml +++ b/metadata/manifest.yml @@ -3,7 +3,8 @@ Cartridge-Short-Name: CELERY Display-Name: Celery Cartridge Description: "A cartridge for running the celeryd daemon to allow asynchronous tasks in your application." Version: '0.1' -Cartridge-Version: 0.0.1 +Cartridge-Version: + - '1.0.1' Cartridge-Vendor: customcarts Source-Url: https://github.com/zzpwelkin/openshift-celery-cartridge.git Categories: From 9b46c8f46b20c067cc6f6a4432811655779690c4 Mon Sep 17 00:00:00 2001 From: zzpwelkin Date: Thu, 26 Dec 2013 23:19:30 +0800 Subject: [PATCH 05/85] Update setup --- bin/setup | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/bin/setup b/bin/setup index 8b13789..e64472e 100755 --- a/bin/setup +++ b/bin/setup @@ -1 +1,2 @@ - +pip install celery==3.0.23 +pip install Scrapy==0.16.1 From 125d9ae49ae7b6c9bc76a3085a5c3990c99f8316 Mon Sep 17 00:00:00 2001 From: zzpwelkin Date: Thu, 26 Dec 2013 23:21:15 +0800 Subject: [PATCH 06/85] Create scrapy.py --- usr/scrapy.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) create mode 100644 usr/scrapy.py diff --git a/usr/scrapy.py b/usr/scrapy.py new file mode 100644 index 0000000..1710a64 --- /dev/null +++ b/usr/scrapy.py @@ -0,0 +1,19 @@ +from scrapy.settings import CrawlerSettings +from scrapy.crawler import CrawlerProcess + +spider_example_name = 'news_spider' + +appname = 'openshift' + +os.environ.setdefault('SCRAPER_SETTINGS_MODULE', '{0}.scraper.settings'.format(appname)) +scrapy_settings_model = __import__(os.getenv('SCRAPER_SETTINGS_MODULE'), fromlist=['']) + + +settings = CrawlerSettings(scrapy_settings_model) +prs = CrawlerProcess(settings) +crawler = prs.create_crawler() +for name, spd in crawler.spiders._spiders.iteritems(): + if name == spider_example_name: + spidercls = spd +crawler.crawl(spidercls(id=1)) +prs.start() From 9fc07f6bf60ce4f5aa7e9b79d52508a881be6b0c Mon Sep 17 00:00:00 2001 From: zzpwelkin Date: Thu, 26 Dec 2013 23:28:15 +0800 Subject: [PATCH 07/85] Update manifest.yml --- metadata/manifest.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/metadata/manifest.yml b/metadata/manifest.yml index 463ab72..a4be907 100644 --- a/metadata/manifest.yml +++ b/metadata/manifest.yml @@ -1,7 +1,7 @@ Name: celeryd Cartridge-Short-Name: CELERY Display-Name: Celery Cartridge -Description: "A cartridge for running the celeryd daemon to allow asynchronous tasks in your application." +Description: A cartridge for running the celeryd daemon to allow asynchronous tasks in your application. Version: '0.1' Cartridge-Version: - '1.0.1' @@ -14,7 +14,7 @@ Provides: Cart-Data: - Key: OPENSHIFT_CELERY_HOME Type: environment - Description: "An environment variable for the Home dir of the binary" + Description: An environment variable for the Home dir of the binary Group-Overrides: - components: - celery From 55826665bbdae80df0cacb574210343287ccb383 Mon Sep 17 00:00:00 2001 From: zzpwelkin Date: Thu, 26 Dec 2013 23:29:56 +0800 Subject: [PATCH 08/85] Update manifest.yml --- metadata/manifest.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/metadata/manifest.yml b/metadata/manifest.yml index a4be907..404d2c5 100644 --- a/metadata/manifest.yml +++ b/metadata/manifest.yml @@ -3,6 +3,8 @@ Cartridge-Short-Name: CELERY Display-Name: Celery Cartridge Description: A cartridge for running the celeryd daemon to allow asynchronous tasks in your application. Version: '0.1' +License: ASL 2.0 +Vendor: zzpwelkin Cartridge-Version: - '1.0.1' Cartridge-Vendor: customcarts From f08e9efb2b04d68caee7c26669611890240ce736 Mon Sep 17 00:00:00 2001 From: zzpwelkin Date: Fri, 27 Dec 2013 09:11:01 +0800 Subject: [PATCH 09/85] PYTHONPATH fixed --- bin/control | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/control b/bin/control index 9681467..888d11a 100755 --- a/bin/control +++ b/bin/control @@ -5,7 +5,7 @@ PATH=/bin/:/usr/bin:$PATH #source $OPENSHIFT_CARTRIDGE_SDK_BASH function start { - export PYTHONPATH=$OPENSHIFT_REPO_DIR/.openshift:$OPENSHIFT_REPO_DIR/wsgi::$OPENSHIFT_CELERY_DIR/conf.d:$OPENSHIFT_REPO_DIR:$PYTHONPATH + export PYTHONPATH=$OPENSHIFT_REPO_DIR/.openshift:$OPENSHIFT_REPO_DIR/wsgi:$OPENSHIFT_CELERY_DIR/conf.d:$OPENSHIFT_REPO_DIR:$PYTHONPATH usr/celeryd-multi start --cmd=celeryd --pidfile=etc/celeryd.pid --hostname=$OPENSHIFT_INTERNAL_IP --loglevel=DEBUG } From 7acaaa7cc844a3cc7fb1a2c7c2d3019dfbfed4fc Mon Sep 17 00:00:00 2001 From: zzpwelkin Date: Fri, 27 Dec 2013 09:15:28 +0800 Subject: [PATCH 10/85] Update setup --- bin/setup | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/setup b/bin/setup index e64472e..18f942a 100755 --- a/bin/setup +++ b/bin/setup @@ -1,2 +1,2 @@ pip install celery==3.0.23 -pip install Scrapy==0.16.1 +#pip install Scrapy==0.16.1 From e35e46c5733a832c4a0de67ca59e558af566dd92 Mon Sep 17 00:00:00 2001 From: zzpwelkin Date: Fri, 27 Dec 2013 09:46:36 +0800 Subject: [PATCH 11/85] Update setup --- bin/setup | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/setup b/bin/setup index 18f942a..68a39d4 100755 --- a/bin/setup +++ b/bin/setup @@ -1,2 +1,2 @@ -pip install celery==3.0.23 +pip install celery==3.0.24 #pip install Scrapy==0.16.1 From f8966c132299491d3681fd07adba97926122d87c Mon Sep 17 00:00:00 2001 From: zzpwelkin Date: Fri, 27 Dec 2013 10:37:10 +0800 Subject: [PATCH 12/85] add absolute path in control --- bin/control | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/control b/bin/control index 888d11a..88eb7d9 100755 --- a/bin/control +++ b/bin/control @@ -6,7 +6,7 @@ PATH=/bin/:/usr/bin:$PATH function start { export PYTHONPATH=$OPENSHIFT_REPO_DIR/.openshift:$OPENSHIFT_REPO_DIR/wsgi:$OPENSHIFT_CELERY_DIR/conf.d:$OPENSHIFT_REPO_DIR:$PYTHONPATH - usr/celeryd-multi start --cmd=celeryd --pidfile=etc/celeryd.pid --hostname=$OPENSHIFT_INTERNAL_IP --loglevel=DEBUG + `echo $OPENSHIFT_CELERY_DIR`usr/celeryd-multi start --cmd=celeryd --pidfile=etc/celeryd.pid --hostname=$OPENSHIFT_INTERNAL_IP --loglevel=DEBUG } function stop { From 2794cb9e98e439aaeef37e87b1b2b79238ae6e7b Mon Sep 17 00:00:00 2001 From: zzpwelkin Date: Fri, 27 Dec 2013 11:17:36 +0800 Subject: [PATCH 13/85] add template dir --- env/OPENSHIFT_CELERY_IMPORTS.erb | 1 + template/.openshift/action_hooks/build | 1 + template/.openshift/action_hooks/deploy | 1 + template/.openshift/action_hooks/post-deploy | 1 + template/.openshift/action_hooks/pre-build | 1 + template/wsgi/celerytks/__init__.py | 0 template/wsgi/celerytks/tasks.py | 6 ++++++ 7 files changed, 11 insertions(+) create mode 100644 env/OPENSHIFT_CELERY_IMPORTS.erb create mode 100644 template/.openshift/action_hooks/build create mode 100644 template/.openshift/action_hooks/deploy create mode 100644 template/.openshift/action_hooks/post-deploy create mode 100644 template/.openshift/action_hooks/pre-build create mode 100644 template/wsgi/celerytks/__init__.py create mode 100644 template/wsgi/celerytks/tasks.py diff --git a/env/OPENSHIFT_CELERY_IMPORTS.erb b/env/OPENSHIFT_CELERY_IMPORTS.erb new file mode 100644 index 0000000..269bf09 --- /dev/null +++ b/env/OPENSHIFT_CELERY_IMPORTS.erb @@ -0,0 +1 @@ +openshift diff --git a/template/.openshift/action_hooks/build b/template/.openshift/action_hooks/build new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/template/.openshift/action_hooks/build @@ -0,0 +1 @@ + diff --git a/template/.openshift/action_hooks/deploy b/template/.openshift/action_hooks/deploy new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/template/.openshift/action_hooks/deploy @@ -0,0 +1 @@ + diff --git a/template/.openshift/action_hooks/post-deploy b/template/.openshift/action_hooks/post-deploy new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/template/.openshift/action_hooks/post-deploy @@ -0,0 +1 @@ + diff --git a/template/.openshift/action_hooks/pre-build b/template/.openshift/action_hooks/pre-build new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/template/.openshift/action_hooks/pre-build @@ -0,0 +1 @@ + diff --git a/template/wsgi/celerytks/__init__.py b/template/wsgi/celerytks/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/template/wsgi/celerytks/tasks.py b/template/wsgi/celerytks/tasks.py new file mode 100644 index 0000000..9e4a85c --- /dev/null +++ b/template/wsgi/celerytks/tasks.py @@ -0,0 +1,6 @@ +import celery +app = celery.Celery() + +@app.task +def addtest(x, y): + return x + y From ac0259276dc209e53fb3515b4afe1f8983468478 Mon Sep 17 00:00:00 2001 From: zzpwelkin Date: Fri, 27 Dec 2013 11:25:46 +0800 Subject: [PATCH 14/85] set python env --- template/.openshift/action_hooks/deploy | 5 +++++ template/.openshift/action_hooks/post_deploy | 4 ++++ 2 files changed, 9 insertions(+) mode change 100644 => 100755 template/.openshift/action_hooks/deploy create mode 100755 template/.openshift/action_hooks/post_deploy diff --git a/template/.openshift/action_hooks/deploy b/template/.openshift/action_hooks/deploy old mode 100644 new mode 100755 index 8b13789..3875983 --- a/template/.openshift/action_hooks/deploy +++ b/template/.openshift/action_hooks/deploy @@ -1 +1,6 @@ +#!/bin/bash +# This deploy hook gets executed after dependencies are resolved and the +# build hook has been run but before the application has been started back +# up again. This script gets executed directly, so it could be python, php, +# ruby, etc. diff --git a/template/.openshift/action_hooks/post_deploy b/template/.openshift/action_hooks/post_deploy new file mode 100755 index 0000000..3295d19 --- /dev/null +++ b/template/.openshift/action_hooks/post_deploy @@ -0,0 +1,4 @@ +#!/bin/bash + +source $OPENSHIFT_HOMEDIR/python/virtenv/bin/activate +export PYTHONPATH=`echo $OPENSHIFT_REPO_DIR`wsgi:$PYTHONPATH From 6fe90ba2528c0d7caf99e8a3fede94780296bab8 Mon Sep 17 00:00:00 2001 From: zzpwelkin Date: Fri, 27 Dec 2013 11:30:51 +0800 Subject: [PATCH 15/85] refact and change the app name --- env/OPENSHIFT_CELERY_IMPORTS.erb | 2 +- template/.openshift/action_hooks/post_deploy | 2 +- template/celerytks/__init__.py | 0 template/celerytks/tasks.py | 6 ++++++ 4 files changed, 8 insertions(+), 2 deletions(-) create mode 100644 template/celerytks/__init__.py create mode 100644 template/celerytks/tasks.py diff --git a/env/OPENSHIFT_CELERY_IMPORTS.erb b/env/OPENSHIFT_CELERY_IMPORTS.erb index 269bf09..1322dda 100644 --- a/env/OPENSHIFT_CELERY_IMPORTS.erb +++ b/env/OPENSHIFT_CELERY_IMPORTS.erb @@ -1 +1 @@ -openshift +celerytks diff --git a/template/.openshift/action_hooks/post_deploy b/template/.openshift/action_hooks/post_deploy index 3295d19..d38c5da 100755 --- a/template/.openshift/action_hooks/post_deploy +++ b/template/.openshift/action_hooks/post_deploy @@ -1,4 +1,4 @@ #!/bin/bash source $OPENSHIFT_HOMEDIR/python/virtenv/bin/activate -export PYTHONPATH=`echo $OPENSHIFT_REPO_DIR`wsgi:$PYTHONPATH +export PYTHONPATH=`echo $OPENSHIFT_REPO_DIR`:$PYTHONPATH diff --git a/template/celerytks/__init__.py b/template/celerytks/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/template/celerytks/tasks.py b/template/celerytks/tasks.py new file mode 100644 index 0000000..9e4a85c --- /dev/null +++ b/template/celerytks/tasks.py @@ -0,0 +1,6 @@ +import celery +app = celery.Celery() + +@app.task +def addtest(x, y): + return x + y From 6ad9eeb655115c7d781c6f99a127add557828f05 Mon Sep 17 00:00:00 2001 From: zzpwelkin Date: Fri, 27 Dec 2013 11:40:12 +0800 Subject: [PATCH 16/85] Update celeryconfig.py.erb --- conf.d/celeryconfig.py.erb | 2 -- 1 file changed, 2 deletions(-) diff --git a/conf.d/celeryconfig.py.erb b/conf.d/celeryconfig.py.erb index 5e5cfc3..bcf2a75 100644 --- a/conf.d/celeryconfig.py.erb +++ b/conf.d/celeryconfig.py.erb @@ -27,8 +27,6 @@ CELERY_RESULT_BACKEND = 'redis://<%= ENV.fetch('OPENSHIFT_CELERY_RESULT_BACKEND' CELERY_IMPORTS = (<%= celery_imports %>, ) -CELERY_ANNOTATIONS = {"tasks.add": {"rate_limit": <%= ENV.fetch('OPENSHIFT_CELERY_RATE_LIM', '10/s') %>}} - """ BROKER_BACKEND = 'mongodb' CELERY_RESULT_BACKEND = 'mongodb' From 2159132cbd59a0dbcbc6e699cb5aaa96df2571e7 Mon Sep 17 00:00:00 2001 From: zzpwelkin Date: Fri, 27 Dec 2013 12:26:24 +0800 Subject: [PATCH 17/85] Update setup --- bin/setup | 1 + 1 file changed, 1 insertion(+) diff --git a/bin/setup b/bin/setup index 68a39d4..67d91c8 100755 --- a/bin/setup +++ b/bin/setup @@ -1,2 +1,3 @@ pip install celery==3.0.24 +pip install pymongo #pip install Scrapy==0.16.1 From 9c65b5119524d62f1aec7678459a528379c9bc7a Mon Sep 17 00:00:00 2001 From: zzpwelkin Date: Fri, 27 Dec 2013 12:33:29 +0800 Subject: [PATCH 18/85] Update celeryconfig.py.erb --- conf.d/celeryconfig.py.erb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conf.d/celeryconfig.py.erb b/conf.d/celeryconfig.py.erb index bcf2a75..06987e2 100644 --- a/conf.d/celeryconfig.py.erb +++ b/conf.d/celeryconfig.py.erb @@ -3,7 +3,7 @@ print 'Configuring celeryd with default cartridge config file' broker_type = ENV.fetch('OPENSHIFT_CELERY_BROKER_TRANS', 'mongodb') if broker_type == "mongodb" %> ## Broker settings. -<% default_url = "localhost:27017/celery_tasks" %> +<% default_url = "ENV.fetch('OPENSHIFT_MONGODB_DB_URL')" %> BROKER_URL = 'mongodb://<%= ENV.fetch('OPENSHIFT_CELERY_BROKER_URL', default_url) %>' ## Using the database to store task state and results. CELERY_RESULT_BACKEND = "mongodb" From 99ff34ad0a07203585f813b421a19760c13c1f28 Mon Sep 17 00:00:00 2001 From: zzpwelkin Date: Fri, 27 Dec 2013 12:41:14 +0800 Subject: [PATCH 19/85] Update celeryconfig.py.erb --- conf.d/celeryconfig.py.erb | 1 + 1 file changed, 1 insertion(+) diff --git a/conf.d/celeryconfig.py.erb b/conf.d/celeryconfig.py.erb index 06987e2..fdc618d 100644 --- a/conf.d/celeryconfig.py.erb +++ b/conf.d/celeryconfig.py.erb @@ -26,6 +26,7 @@ CELERY_RESULT_BACKEND = 'redis://<%= ENV.fetch('OPENSHIFT_CELERY_RESULT_BACKEND' %> CELERY_IMPORTS = (<%= celery_imports %>, ) +CELERYD_LOG_FILE="<%= ENV.fetch('OPENSHIFT_TMP_DIR')/log/celery/worker1.log" """ BROKER_BACKEND = 'mongodb' From 516ac5c9f93b1e17515ac7f5cb8d5275b01dcf53 Mon Sep 17 00:00:00 2001 From: zzpwelkin Date: Fri, 27 Dec 2013 13:33:42 +0800 Subject: [PATCH 20/85] Update celeryconfig.py.erb --- conf.d/celeryconfig.py.erb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conf.d/celeryconfig.py.erb b/conf.d/celeryconfig.py.erb index fdc618d..a3cd479 100644 --- a/conf.d/celeryconfig.py.erb +++ b/conf.d/celeryconfig.py.erb @@ -3,7 +3,7 @@ print 'Configuring celeryd with default cartridge config file' broker_type = ENV.fetch('OPENSHIFT_CELERY_BROKER_TRANS', 'mongodb') if broker_type == "mongodb" %> ## Broker settings. -<% default_url = "ENV.fetch('OPENSHIFT_MONGODB_DB_URL')" %> +<% default_url = ENV.fetch('OPENSHIFT_MONGODB_DB_URL') %> BROKER_URL = 'mongodb://<%= ENV.fetch('OPENSHIFT_CELERY_BROKER_URL', default_url) %>' ## Using the database to store task state and results. CELERY_RESULT_BACKEND = "mongodb" From 7b55b22ab43ba2ba33c2854be45a37d8ca67302a Mon Sep 17 00:00:00 2001 From: zzpwelkin Date: Fri, 27 Dec 2013 13:44:18 +0800 Subject: [PATCH 21/85] Update celeryconfig.py.erb --- conf.d/celeryconfig.py.erb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conf.d/celeryconfig.py.erb b/conf.d/celeryconfig.py.erb index a3cd479..abdc6ad 100644 --- a/conf.d/celeryconfig.py.erb +++ b/conf.d/celeryconfig.py.erb @@ -3,7 +3,7 @@ print 'Configuring celeryd with default cartridge config file' broker_type = ENV.fetch('OPENSHIFT_CELERY_BROKER_TRANS', 'mongodb') if broker_type == "mongodb" %> ## Broker settings. -<% default_url = ENV.fetch('OPENSHIFT_MONGODB_DB_URL') %> +<% default_url = ENV.fetch('OPENSHIFT_MONGODB_DB_URL', 'mongodb://localshot:12607') %> BROKER_URL = 'mongodb://<%= ENV.fetch('OPENSHIFT_CELERY_BROKER_URL', default_url) %>' ## Using the database to store task state and results. CELERY_RESULT_BACKEND = "mongodb" From 9533d9acf343e5cfc6c2d959e500f389d3eab114 Mon Sep 17 00:00:00 2001 From: zzpwelkin Date: Fri, 27 Dec 2013 14:02:30 +0800 Subject: [PATCH 22/85] Update celeryconfig.py.erb --- conf.d/celeryconfig.py.erb | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/conf.d/celeryconfig.py.erb b/conf.d/celeryconfig.py.erb index abdc6ad..d09c889 100644 --- a/conf.d/celeryconfig.py.erb +++ b/conf.d/celeryconfig.py.erb @@ -3,11 +3,11 @@ print 'Configuring celeryd with default cartridge config file' broker_type = ENV.fetch('OPENSHIFT_CELERY_BROKER_TRANS', 'mongodb') if broker_type == "mongodb" %> ## Broker settings. -<% default_url = ENV.fetch('OPENSHIFT_MONGODB_DB_URL', 'mongodb://localshot:12607') %> -BROKER_URL = 'mongodb://<%= ENV.fetch('OPENSHIFT_CELERY_BROKER_URL', default_url) %>' +<% default_url = 'mongodb://localshot:12607'%> +<% BROKER_URL = ENV.fetch('OPENSHIFT_CELERY_BROKER_URL', default_url) %> ## Using the database to store task state and results. CELERY_RESULT_BACKEND = "mongodb" -CELERY_RESULT_DBURI = 'mongodb://<%= ENV.fetch('OPENSHIFT_CELERY_RESULT_BACKEND', default_url) %>' +<% CELERY_RESULT_DBURI = ENV.fetch('OPENSHIFT_CELERY_RESULT_BACKEND', default_url) %> <% elsif broker_type == "amqp" %> ## Broker settings From fc717dd27a3bedaaf296b79134b184d14d0f90fa Mon Sep 17 00:00:00 2001 From: zzpwelkin Date: Fri, 27 Dec 2013 14:03:08 +0800 Subject: [PATCH 23/85] add env --- env/OPENSHIFT_CELERY_BROKER_TRANS.erb | 1 + env/OPENSHIFT_CELERY_BROKER_URL.erb | 2 ++ env/OPENSHIFT_CELERY_RESULT_BACKEND.erb | 1 + template.git/.openshift/action_hooks/build | 1 + template.git/.openshift/action_hooks/deploy | 6 ++++++ template.git/.openshift/action_hooks/post-deploy | 1 + template.git/.openshift/action_hooks/post_deploy | 4 ++++ template.git/.openshift/action_hooks/pre-build | 1 + template.git/celerytks/__init__.py | 0 template.git/celerytks/tasks.py | 13 +++++++++++++ 10 files changed, 30 insertions(+) create mode 100644 env/OPENSHIFT_CELERY_BROKER_TRANS.erb create mode 100644 env/OPENSHIFT_CELERY_BROKER_URL.erb create mode 100644 env/OPENSHIFT_CELERY_RESULT_BACKEND.erb create mode 100644 template.git/.openshift/action_hooks/build create mode 100755 template.git/.openshift/action_hooks/deploy create mode 100644 template.git/.openshift/action_hooks/post-deploy create mode 100755 template.git/.openshift/action_hooks/post_deploy create mode 100644 template.git/.openshift/action_hooks/pre-build create mode 100644 template.git/celerytks/__init__.py create mode 100644 template.git/celerytks/tasks.py diff --git a/env/OPENSHIFT_CELERY_BROKER_TRANS.erb b/env/OPENSHIFT_CELERY_BROKER_TRANS.erb new file mode 100644 index 0000000..97ee1da --- /dev/null +++ b/env/OPENSHIFT_CELERY_BROKER_TRANS.erb @@ -0,0 +1 @@ +mongodb diff --git a/env/OPENSHIFT_CELERY_BROKER_URL.erb b/env/OPENSHIFT_CELERY_BROKER_URL.erb new file mode 100644 index 0000000..b1ae0ea --- /dev/null +++ b/env/OPENSHIFT_CELERY_BROKER_URL.erb @@ -0,0 +1,2 @@ +<%= ENV['OPENSHIFT_MONGODB_DB_URL'] %> + diff --git a/env/OPENSHIFT_CELERY_RESULT_BACKEND.erb b/env/OPENSHIFT_CELERY_RESULT_BACKEND.erb new file mode 100644 index 0000000..33ab847 --- /dev/null +++ b/env/OPENSHIFT_CELERY_RESULT_BACKEND.erb @@ -0,0 +1 @@ +<%= ENV['OPENSHIFT_MONGODB_DB_URL'] %> diff --git a/template.git/.openshift/action_hooks/build b/template.git/.openshift/action_hooks/build new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/template.git/.openshift/action_hooks/build @@ -0,0 +1 @@ + diff --git a/template.git/.openshift/action_hooks/deploy b/template.git/.openshift/action_hooks/deploy new file mode 100755 index 0000000..3875983 --- /dev/null +++ b/template.git/.openshift/action_hooks/deploy @@ -0,0 +1,6 @@ +#!/bin/bash +# This deploy hook gets executed after dependencies are resolved and the +# build hook has been run but before the application has been started back +# up again. This script gets executed directly, so it could be python, php, +# ruby, etc. + diff --git a/template.git/.openshift/action_hooks/post-deploy b/template.git/.openshift/action_hooks/post-deploy new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/template.git/.openshift/action_hooks/post-deploy @@ -0,0 +1 @@ + diff --git a/template.git/.openshift/action_hooks/post_deploy b/template.git/.openshift/action_hooks/post_deploy new file mode 100755 index 0000000..d38c5da --- /dev/null +++ b/template.git/.openshift/action_hooks/post_deploy @@ -0,0 +1,4 @@ +#!/bin/bash + +source $OPENSHIFT_HOMEDIR/python/virtenv/bin/activate +export PYTHONPATH=`echo $OPENSHIFT_REPO_DIR`:$PYTHONPATH diff --git a/template.git/.openshift/action_hooks/pre-build b/template.git/.openshift/action_hooks/pre-build new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/template.git/.openshift/action_hooks/pre-build @@ -0,0 +1 @@ + diff --git a/template.git/celerytks/__init__.py b/template.git/celerytks/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/template.git/celerytks/tasks.py b/template.git/celerytks/tasks.py new file mode 100644 index 0000000..2041943 --- /dev/null +++ b/template.git/celerytks/tasks.py @@ -0,0 +1,13 @@ +import celery +app = celery.Celery() + +@app.task +def addtest(x, y): + return x + y + +@app.task +def run_spiders(name, **kwargs): + """ + @param name: spider name + """ + pass From eae61b8060af884cdf2eed84dd257e2ca7c54273 Mon Sep 17 00:00:00 2001 From: zzpwelkin Date: Fri, 27 Dec 2013 14:48:51 +0800 Subject: [PATCH 24/85] change erb file --- conf.d/celeryconfig.py.erb | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/conf.d/celeryconfig.py.erb b/conf.d/celeryconfig.py.erb index d09c889..3d78e01 100644 --- a/conf.d/celeryconfig.py.erb +++ b/conf.d/celeryconfig.py.erb @@ -3,11 +3,11 @@ print 'Configuring celeryd with default cartridge config file' broker_type = ENV.fetch('OPENSHIFT_CELERY_BROKER_TRANS', 'mongodb') if broker_type == "mongodb" %> ## Broker settings. -<% default_url = 'mongodb://localshot:12607'%> -<% BROKER_URL = ENV.fetch('OPENSHIFT_CELERY_BROKER_URL', default_url) %> +default_url = 'mongodb://localshot:12607' +BROKER_URL = <%= ENV.fetch('OPENSHIFT_CELERY_BROKER_URL', default_url) %> ## Using the database to store task state and results. CELERY_RESULT_BACKEND = "mongodb" -<% CELERY_RESULT_DBURI = ENV.fetch('OPENSHIFT_CELERY_RESULT_BACKEND', default_url) %> +CELERY_RESULT_DBURI = <%= ENV.fetch('OPENSHIFT_CELERY_RESULT_BACKEND', default_url) %> <% elsif broker_type == "amqp" %> ## Broker settings @@ -26,7 +26,7 @@ CELERY_RESULT_BACKEND = 'redis://<%= ENV.fetch('OPENSHIFT_CELERY_RESULT_BACKEND' %> CELERY_IMPORTS = (<%= celery_imports %>, ) -CELERYD_LOG_FILE="<%= ENV.fetch('OPENSHIFT_TMP_DIR')/log/celery/worker1.log" +CELERYD_LOG_FILE = '<%= ENV.fetch('OPENSHIFT_TMP_DIR')/log/celery/worker1.log' """ BROKER_BACKEND = 'mongodb' From 29a14b903f6b91104987a11f51e4a4fd26c10fe6 Mon Sep 17 00:00:00 2001 From: zzpwelkin Date: Fri, 27 Dec 2013 14:52:07 +0800 Subject: [PATCH 25/85] fix --- conf.d/celeryconfig.py.erb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conf.d/celeryconfig.py.erb b/conf.d/celeryconfig.py.erb index 3d78e01..201cbfd 100644 --- a/conf.d/celeryconfig.py.erb +++ b/conf.d/celeryconfig.py.erb @@ -26,7 +26,7 @@ CELERY_RESULT_BACKEND = 'redis://<%= ENV.fetch('OPENSHIFT_CELERY_RESULT_BACKEND' %> CELERY_IMPORTS = (<%= celery_imports %>, ) -CELERYD_LOG_FILE = '<%= ENV.fetch('OPENSHIFT_TMP_DIR')/log/celery/worker1.log' +CELERYD_LOG_FILE = '<%= ENV.fetch('OPENSHIFT_TMP_DIR') %>log/celery/worker1.log' """ BROKER_BACKEND = 'mongodb' From 8b0ea903c5b23c9d46832914cf357e66c1567fda Mon Sep 17 00:00:00 2001 From: zzpwelkin Date: Fri, 27 Dec 2013 15:04:16 +0800 Subject: [PATCH 26/85] fix --- conf.d/celeryconfig.py.erb | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/conf.d/celeryconfig.py.erb b/conf.d/celeryconfig.py.erb index 201cbfd..73b1374 100644 --- a/conf.d/celeryconfig.py.erb +++ b/conf.d/celeryconfig.py.erb @@ -3,11 +3,11 @@ print 'Configuring celeryd with default cartridge config file' broker_type = ENV.fetch('OPENSHIFT_CELERY_BROKER_TRANS', 'mongodb') if broker_type == "mongodb" %> ## Broker settings. -default_url = 'mongodb://localshot:12607' -BROKER_URL = <%= ENV.fetch('OPENSHIFT_CELERY_BROKER_URL', default_url) %> +##default_url = 'mongodb://localshot:12607' +BROKER_URL = '<%= ENV.fetch('OPENSHIFT_CELERY_BROKER_URL', 'mongodb://localshot:12607') %>' ## Using the database to store task state and results. CELERY_RESULT_BACKEND = "mongodb" -CELERY_RESULT_DBURI = <%= ENV.fetch('OPENSHIFT_CELERY_RESULT_BACKEND', default_url) %> +CELERY_RESULT_DBURI = '<%= ENV.fetch('OPENSHIFT_CELERY_RESULT_BACKEND', 'mongodb://localshot:12607') %>' <% elsif broker_type == "amqp" %> ## Broker settings From a713eda82bc8863219d2491d3f00a0b17dea423e Mon Sep 17 00:00:00 2001 From: zzpwelkin Date: Fri, 27 Dec 2013 15:21:48 +0800 Subject: [PATCH 27/85] configure file --- conf.d/celeryconfig.py | 8 +++++ conf.d/celeryconfig.py.erb.bg | 65 +++++++++++++++++++++++++++++++++++ 2 files changed, 73 insertions(+) create mode 100644 conf.d/celeryconfig.py create mode 100644 conf.d/celeryconfig.py.erb.bg diff --git a/conf.d/celeryconfig.py b/conf.d/celeryconfig.py new file mode 100644 index 0000000..8058cf4 --- /dev/null +++ b/conf.d/celeryconfig.py @@ -0,0 +1,8 @@ +import os + +BROKER_URL = os.environ.get('OPENSHIFT_MONGODB_DB_URL') +CELERY_RESULT_BACKEND = "mongodb" +CELERY_RESULT_DBURI = os.environ.get('OPENSHIFT_MONGODB_DB_URL') + +CELERY_IMPORTS = ('celerytks', ) +CELERYD_LOG_FILE = '{0}log/celery/worker1.log'.format(os.environ.get('OPENSHIFT_TMP_DIR')) diff --git a/conf.d/celeryconfig.py.erb.bg b/conf.d/celeryconfig.py.erb.bg new file mode 100644 index 0000000..73b1374 --- /dev/null +++ b/conf.d/celeryconfig.py.erb.bg @@ -0,0 +1,65 @@ +print 'Configuring celeryd with default cartridge config file' +<% +broker_type = ENV.fetch('OPENSHIFT_CELERY_BROKER_TRANS', 'mongodb') +if broker_type == "mongodb" %> +## Broker settings. +##default_url = 'mongodb://localshot:12607' +BROKER_URL = '<%= ENV.fetch('OPENSHIFT_CELERY_BROKER_URL', 'mongodb://localshot:12607') %>' +## Using the database to store task state and results. +CELERY_RESULT_BACKEND = "mongodb" +CELERY_RESULT_DBURI = '<%= ENV.fetch('OPENSHIFT_CELERY_RESULT_BACKEND', 'mongodb://localshot:12607') %>' + +<% elsif broker_type == "amqp" %> +## Broker settings +BROKER_URL = 'amqp://<%= ENV.fetch('OPENSHIFT_CELERY_BROKER_URL', 'guest:guest@localhost:5672//') %>' +<% elsif broker_type == "redis" %> +## Broker settings +BROKER_URL = 'redis://<%= ENV.fetch('OPENSHIFT_CELERY_BROKER_URL', 'localhost:6379/0') %>' +BROKER_TRANSPORT_OPTIONS = {'visibility_timeout': <%= ENV.fetch('OPENSHIFT_CELERY_VISIBILITY_TIME', 3600) %> } +CELERY_RESULT_BACKEND = 'redis://<%= ENV.fetch('OPENSHIFT_CELERY_RESULT_BACKEND', 'localhost:6379/0') %>' +<% end %> +# List of modules to import when celery starts. +<% + celery_imports = ENV.fetch('OPENSHIFT_CELERY_IMPORTS', '').split(',') + celery_imports = celery_imports.map { |a| "'" + a + "'" } + celery_imports = celery_imports.join(",") +%> +CELERY_IMPORTS = (<%= celery_imports %>, ) + +CELERYD_LOG_FILE = '<%= ENV.fetch('OPENSHIFT_TMP_DIR') %>log/celery/worker1.log' + +""" +BROKER_BACKEND = 'mongodb' +CELERY_RESULT_BACKEND = 'mongodb' +BROKER_HOST = os.environ.get('OPENSHIFT_MONGODB_DB_HOST') +BROKER_PORT = os.environ.get('OPENSHIFT_MONGODB_DB_PORT') +BROKER_USER = os.environ.get('OPENSHIFT_MONGODB_DB_USERNAME') +BROKER_PASSWORD = os.environ.get('OPENSHIFT_MONGODB_DB_PASSWORD') +BROKER_VHOST = 'celery_tasks' +BROKER_URL = 'mongodb://admin:XCf67tq_BqKU@writeown-tresback.rhcloud.com/celery_tasks' +CELERY_RESULT_BACKEND = 'mongodb://admin:XCf67tq_BqKU@writeown-tresback.rhcloud.com/celery_tasks' +BROKER_URL = 'mongodb://admin:XCf67tq_BqKU@127.8.167.130/celery_tasks' +#"%scelery_tasks" % os.environ.get('OPENSHIFT_MONGODB_DB_URL') +BROKER_BACKEND = 'mongodb' +CELERY_RESULT_BACKEND = 'mongodb' +BROKER_HOST = os.environ.get('OPENSHIFT_MONGODB_DB_HOST') +BROKER_PORT = os.environ.get('OPENSHIFT_MONGODB_DB_PORT') +BROKER_TRANSPORT = 'mongodb' +BROKER_VHOST = 'celery_tasks' +CELERY_MONGODB_BACKEND_SETTINGS = { + "host": os.environ.get('OPENSHIFT_MONGODB_DB_HOST'), + "port": os.environ.get('OPENSHIFT_MONGODB_DB_PORT'), + "database": 'celery_tasks', +} +CELERY_MONGODB_BACKEND_SETTINGS = { + "host": os.environ.get('OPENSHIFT_MONGODB_DB_HOST'), + "port": os.environ.get('OPENSHIFT_MONGODB_DB_PORT'), + "database": 'celery_tasks', + "user": os.environ.get('OPENSHIFT_MONGODB_DB_USERNAME'), + "password": os.environ.get('OPENSHIFT_MONGODB_DB_PASSWORD') +} + +BROKER_BACKEND = 'redis' +BROKER_URL = "redis://127.8.167.131:16379/0" +CELERY_RESULT_BACKEND = "redis://127.8.167.131:16379/0" +""" From 381a916fae258f288ceb423e43ba72d6f26259ba Mon Sep 17 00:00:00 2001 From: zzpwelkin Date: Fri, 27 Dec 2013 15:23:51 +0800 Subject: [PATCH 28/85] clear --- conf.d/celeryconfig.py.erb | 65 ----------------------------- env/OPENSHIFT_CELERY_BROKER_URL.erb | 1 - template/celerytks/tasks.py | 7 ++++ template/wsgi/celerytks/__init__.py | 0 template/wsgi/celerytks/tasks.py | 6 --- 5 files changed, 7 insertions(+), 72 deletions(-) delete mode 100644 conf.d/celeryconfig.py.erb delete mode 100644 template/wsgi/celerytks/__init__.py delete mode 100644 template/wsgi/celerytks/tasks.py diff --git a/conf.d/celeryconfig.py.erb b/conf.d/celeryconfig.py.erb deleted file mode 100644 index 73b1374..0000000 --- a/conf.d/celeryconfig.py.erb +++ /dev/null @@ -1,65 +0,0 @@ -print 'Configuring celeryd with default cartridge config file' -<% -broker_type = ENV.fetch('OPENSHIFT_CELERY_BROKER_TRANS', 'mongodb') -if broker_type == "mongodb" %> -## Broker settings. -##default_url = 'mongodb://localshot:12607' -BROKER_URL = '<%= ENV.fetch('OPENSHIFT_CELERY_BROKER_URL', 'mongodb://localshot:12607') %>' -## Using the database to store task state and results. -CELERY_RESULT_BACKEND = "mongodb" -CELERY_RESULT_DBURI = '<%= ENV.fetch('OPENSHIFT_CELERY_RESULT_BACKEND', 'mongodb://localshot:12607') %>' - -<% elsif broker_type == "amqp" %> -## Broker settings -BROKER_URL = 'amqp://<%= ENV.fetch('OPENSHIFT_CELERY_BROKER_URL', 'guest:guest@localhost:5672//') %>' -<% elsif broker_type == "redis" %> -## Broker settings -BROKER_URL = 'redis://<%= ENV.fetch('OPENSHIFT_CELERY_BROKER_URL', 'localhost:6379/0') %>' -BROKER_TRANSPORT_OPTIONS = {'visibility_timeout': <%= ENV.fetch('OPENSHIFT_CELERY_VISIBILITY_TIME', 3600) %> } -CELERY_RESULT_BACKEND = 'redis://<%= ENV.fetch('OPENSHIFT_CELERY_RESULT_BACKEND', 'localhost:6379/0') %>' -<% end %> -# List of modules to import when celery starts. -<% - celery_imports = ENV.fetch('OPENSHIFT_CELERY_IMPORTS', '').split(',') - celery_imports = celery_imports.map { |a| "'" + a + "'" } - celery_imports = celery_imports.join(",") -%> -CELERY_IMPORTS = (<%= celery_imports %>, ) - -CELERYD_LOG_FILE = '<%= ENV.fetch('OPENSHIFT_TMP_DIR') %>log/celery/worker1.log' - -""" -BROKER_BACKEND = 'mongodb' -CELERY_RESULT_BACKEND = 'mongodb' -BROKER_HOST = os.environ.get('OPENSHIFT_MONGODB_DB_HOST') -BROKER_PORT = os.environ.get('OPENSHIFT_MONGODB_DB_PORT') -BROKER_USER = os.environ.get('OPENSHIFT_MONGODB_DB_USERNAME') -BROKER_PASSWORD = os.environ.get('OPENSHIFT_MONGODB_DB_PASSWORD') -BROKER_VHOST = 'celery_tasks' -BROKER_URL = 'mongodb://admin:XCf67tq_BqKU@writeown-tresback.rhcloud.com/celery_tasks' -CELERY_RESULT_BACKEND = 'mongodb://admin:XCf67tq_BqKU@writeown-tresback.rhcloud.com/celery_tasks' -BROKER_URL = 'mongodb://admin:XCf67tq_BqKU@127.8.167.130/celery_tasks' -#"%scelery_tasks" % os.environ.get('OPENSHIFT_MONGODB_DB_URL') -BROKER_BACKEND = 'mongodb' -CELERY_RESULT_BACKEND = 'mongodb' -BROKER_HOST = os.environ.get('OPENSHIFT_MONGODB_DB_HOST') -BROKER_PORT = os.environ.get('OPENSHIFT_MONGODB_DB_PORT') -BROKER_TRANSPORT = 'mongodb' -BROKER_VHOST = 'celery_tasks' -CELERY_MONGODB_BACKEND_SETTINGS = { - "host": os.environ.get('OPENSHIFT_MONGODB_DB_HOST'), - "port": os.environ.get('OPENSHIFT_MONGODB_DB_PORT'), - "database": 'celery_tasks', -} -CELERY_MONGODB_BACKEND_SETTINGS = { - "host": os.environ.get('OPENSHIFT_MONGODB_DB_HOST'), - "port": os.environ.get('OPENSHIFT_MONGODB_DB_PORT'), - "database": 'celery_tasks', - "user": os.environ.get('OPENSHIFT_MONGODB_DB_USERNAME'), - "password": os.environ.get('OPENSHIFT_MONGODB_DB_PASSWORD') -} - -BROKER_BACKEND = 'redis' -BROKER_URL = "redis://127.8.167.131:16379/0" -CELERY_RESULT_BACKEND = "redis://127.8.167.131:16379/0" -""" diff --git a/env/OPENSHIFT_CELERY_BROKER_URL.erb b/env/OPENSHIFT_CELERY_BROKER_URL.erb index b1ae0ea..33ab847 100644 --- a/env/OPENSHIFT_CELERY_BROKER_URL.erb +++ b/env/OPENSHIFT_CELERY_BROKER_URL.erb @@ -1,2 +1 @@ <%= ENV['OPENSHIFT_MONGODB_DB_URL'] %> - diff --git a/template/celerytks/tasks.py b/template/celerytks/tasks.py index 9e4a85c..2041943 100644 --- a/template/celerytks/tasks.py +++ b/template/celerytks/tasks.py @@ -4,3 +4,10 @@ @app.task def addtest(x, y): return x + y + +@app.task +def run_spiders(name, **kwargs): + """ + @param name: spider name + """ + pass diff --git a/template/wsgi/celerytks/__init__.py b/template/wsgi/celerytks/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/template/wsgi/celerytks/tasks.py b/template/wsgi/celerytks/tasks.py deleted file mode 100644 index 9e4a85c..0000000 --- a/template/wsgi/celerytks/tasks.py +++ /dev/null @@ -1,6 +0,0 @@ -import celery -app = celery.Celery() - -@app.task -def addtest(x, y): - return x + y From e05c600b0cf79ec38ae470b1182cbd71a9712de6 Mon Sep 17 00:00:00 2001 From: zzpwelkin Date: Fri, 27 Dec 2013 15:36:12 +0800 Subject: [PATCH 29/85] fix --- conf.d/celeryconfig.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conf.d/celeryconfig.py b/conf.d/celeryconfig.py index 8058cf4..dd27286 100644 --- a/conf.d/celeryconfig.py +++ b/conf.d/celeryconfig.py @@ -5,4 +5,4 @@ CELERY_RESULT_DBURI = os.environ.get('OPENSHIFT_MONGODB_DB_URL') CELERY_IMPORTS = ('celerytks', ) -CELERYD_LOG_FILE = '{0}log/celery/worker1.log'.format(os.environ.get('OPENSHIFT_TMP_DIR')) +#CELERYD_LOG_FILE = '{0}log/celery/worker1.log'.format(os.environ.get('OPENSHIFT_TMP_DIR')) From 9415e8b72319e2934aadc4c42ad4cb7d49caf6f1 Mon Sep 17 00:00:00 2001 From: zzpwelkin Date: Fri, 27 Dec 2013 16:08:29 +0800 Subject: [PATCH 30/85] fix error --- bin/control | 2 +- conf.d/celeryconfig.py | 7 +++++-- template/celerytks/tasks.py | 1 + 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/bin/control b/bin/control index 88eb7d9..b112a5c 100755 --- a/bin/control +++ b/bin/control @@ -6,7 +6,7 @@ PATH=/bin/:/usr/bin:$PATH function start { export PYTHONPATH=$OPENSHIFT_REPO_DIR/.openshift:$OPENSHIFT_REPO_DIR/wsgi:$OPENSHIFT_CELERY_DIR/conf.d:$OPENSHIFT_REPO_DIR:$PYTHONPATH - `echo $OPENSHIFT_CELERY_DIR`usr/celeryd-multi start --cmd=celeryd --pidfile=etc/celeryd.pid --hostname=$OPENSHIFT_INTERNAL_IP --loglevel=DEBUG + `echo $OPENSHIFT_CELERY_DIR`usr/celeryd-multi start --cmd=celeryd --pidfile=`echo $OPENSHIFT_CELERY_DIR`etc/celeryd.pid --hostname=$OPENSHIFT_INTERNAL_IP --loglevel=DEBUG } function stop { diff --git a/conf.d/celeryconfig.py b/conf.d/celeryconfig.py index dd27286..b453cf4 100644 --- a/conf.d/celeryconfig.py +++ b/conf.d/celeryconfig.py @@ -4,5 +4,8 @@ CELERY_RESULT_BACKEND = "mongodb" CELERY_RESULT_DBURI = os.environ.get('OPENSHIFT_MONGODB_DB_URL') -CELERY_IMPORTS = ('celerytks', ) -#CELERYD_LOG_FILE = '{0}log/celery/worker1.log'.format(os.environ.get('OPENSHIFT_TMP_DIR')) +celery_imports = os.environ.get('OPENSHIFT_CELERY_IMPORTS') + +CELERY_IMPORTS = celery_import.split(',') if celery_imports else ('celerytks',) + +CELERYD_LOG_FILE = '{0}log/celery/worker1.log'.format(os.environ.get('OPENSHIFT_TMP_DIR')) diff --git a/template/celerytks/tasks.py b/template/celerytks/tasks.py index 2041943..9f779ee 100644 --- a/template/celerytks/tasks.py +++ b/template/celerytks/tasks.py @@ -1,5 +1,6 @@ import celery app = celery.Celery() +app.config_from_object('celeryconfig') @app.task def addtest(x, y): From f1b72f81396edff9485b79d4bef76550e8fdb013 Mon Sep 17 00:00:00 2001 From: zzpwelkin Date: Fri, 27 Dec 2013 18:03:10 +0800 Subject: [PATCH 31/85] Update celeryconfig.py --- conf.d/celeryconfig.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conf.d/celeryconfig.py b/conf.d/celeryconfig.py index b453cf4..d042a71 100644 --- a/conf.d/celeryconfig.py +++ b/conf.d/celeryconfig.py @@ -6,6 +6,6 @@ celery_imports = os.environ.get('OPENSHIFT_CELERY_IMPORTS') -CELERY_IMPORTS = celery_import.split(',') if celery_imports else ('celerytks',) +CELERY_IMPORTS = celery_imports.split(',') if celery_imports else ('celerytks',) CELERYD_LOG_FILE = '{0}log/celery/worker1.log'.format(os.environ.get('OPENSHIFT_TMP_DIR')) From 7e04c171fcf9de772509d2795f7e577e219b7a02 Mon Sep 17 00:00:00 2001 From: zzpwelkin Date: Fri, 27 Dec 2013 21:58:16 +0800 Subject: [PATCH 32/85] merge --- bin/control | 2 +- bin/setup | 1 - conf.d/celeryconfig.py | 3 +-- 3 files changed, 2 insertions(+), 4 deletions(-) diff --git a/bin/control b/bin/control index b112a5c..3e07af1 100755 --- a/bin/control +++ b/bin/control @@ -6,7 +6,7 @@ PATH=/bin/:/usr/bin:$PATH function start { export PYTHONPATH=$OPENSHIFT_REPO_DIR/.openshift:$OPENSHIFT_REPO_DIR/wsgi:$OPENSHIFT_CELERY_DIR/conf.d:$OPENSHIFT_REPO_DIR:$PYTHONPATH - `echo $OPENSHIFT_CELERY_DIR`usr/celeryd-multi start --cmd=celeryd --pidfile=`echo $OPENSHIFT_CELERY_DIR`etc/celeryd.pid --hostname=$OPENSHIFT_INTERNAL_IP --loglevel=DEBUG + `echo $OPENSHIFT_CELERY_DIR`usr/celeryd-multi start --cmd=celeryd --pidfile=`echo $OPENSHIFT_CELERY_DIR`etc/celeryd.pid --hostname=$OPENSHIFT_INTERNAL_IP --loglevel=DEBUG `echo $OPENFHIST_TMP_DIR`celery_log.txt } function stop { diff --git a/bin/setup b/bin/setup index 67d91c8..2b8d9f4 100755 --- a/bin/setup +++ b/bin/setup @@ -1,3 +1,2 @@ pip install celery==3.0.24 pip install pymongo -#pip install Scrapy==0.16.1 diff --git a/conf.d/celeryconfig.py b/conf.d/celeryconfig.py index d042a71..cc77ba8 100644 --- a/conf.d/celeryconfig.py +++ b/conf.d/celeryconfig.py @@ -6,6 +6,5 @@ celery_imports = os.environ.get('OPENSHIFT_CELERY_IMPORTS') -CELERY_IMPORTS = celery_imports.split(',') if celery_imports else ('celerytks',) +CELERY_IMPORTS = celery_imports.split(',') if celery_imports else ('celerytks.tasks',) -CELERYD_LOG_FILE = '{0}log/celery/worker1.log'.format(os.environ.get('OPENSHIFT_TMP_DIR')) From 9b55ad1152387ac71f6cfea5b2dcfb08a8b6d620 Mon Sep 17 00:00:00 2001 From: zzpwelkin Date: Fri, 27 Dec 2013 22:17:28 +0800 Subject: [PATCH 33/85] fix stop error of this script --- bin/control | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/control b/bin/control index 3e07af1..f995117 100755 --- a/bin/control +++ b/bin/control @@ -10,7 +10,7 @@ function start { } function stop { - if ps -p `cat etc/celeryd.pid` > /dev/null 2>$1 + if ps -p `cat etc/celeryd.pid` > /dev/null; then kill -9 `cat etc/celeryd.pid` else From 44cd908a994a70b020abcf6c1513fc9fb05ad075 Mon Sep 17 00:00:00 2001 From: zzpwelkin Date: Fri, 27 Dec 2013 22:40:06 +0800 Subject: [PATCH 34/85] celery imports fixed --- bin/control | 2 +- conf.d/celeryconfig.py | 2 +- env/OPENSHIFT_CELERY_IMPORTS.erb | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/bin/control b/bin/control index f995117..0bf59b7 100755 --- a/bin/control +++ b/bin/control @@ -6,7 +6,7 @@ PATH=/bin/:/usr/bin:$PATH function start { export PYTHONPATH=$OPENSHIFT_REPO_DIR/.openshift:$OPENSHIFT_REPO_DIR/wsgi:$OPENSHIFT_CELERY_DIR/conf.d:$OPENSHIFT_REPO_DIR:$PYTHONPATH - `echo $OPENSHIFT_CELERY_DIR`usr/celeryd-multi start --cmd=celeryd --pidfile=`echo $OPENSHIFT_CELERY_DIR`etc/celeryd.pid --hostname=$OPENSHIFT_INTERNAL_IP --loglevel=DEBUG `echo $OPENFHIST_TMP_DIR`celery_log.txt + `echo $OPENSHIFT_CELERY_DIR`usr/celeryd-multi start --cmd=celeryd --pidfile=`echo $OPENSHIFT_CELERY_DIR`etc/celeryd.pid --hostname=$OPENSHIFT_INTERNAL_IP --loglevel=DEBUG `echo $OPENSHIFT_TMP_DIR`celery_log.txt } function stop { diff --git a/conf.d/celeryconfig.py b/conf.d/celeryconfig.py index cc77ba8..1c0768c 100644 --- a/conf.d/celeryconfig.py +++ b/conf.d/celeryconfig.py @@ -6,5 +6,5 @@ celery_imports = os.environ.get('OPENSHIFT_CELERY_IMPORTS') -CELERY_IMPORTS = celery_imports.split(',') if celery_imports else ('celerytks.tasks',) +CELERY_IMPORTS = celery_imports.split(',') diff --git a/env/OPENSHIFT_CELERY_IMPORTS.erb b/env/OPENSHIFT_CELERY_IMPORTS.erb index 1322dda..c9ead1e 100644 --- a/env/OPENSHIFT_CELERY_IMPORTS.erb +++ b/env/OPENSHIFT_CELERY_IMPORTS.erb @@ -1 +1 @@ -celerytks +celerytks.tasks From 0848e25b014e213b1cb3a26892e790b2fcb79f83 Mon Sep 17 00:00:00 2001 From: zzpwelkin Date: Wed, 1 Jan 2014 12:58:28 +0800 Subject: [PATCH 35/85] [FIX] reset env and start error --- bin/control | 2 +- env/OPENSHIFT_CELERY_IMPORTS.erb | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/bin/control b/bin/control index 0bf59b7..74faca5 100755 --- a/bin/control +++ b/bin/control @@ -6,7 +6,7 @@ PATH=/bin/:/usr/bin:$PATH function start { export PYTHONPATH=$OPENSHIFT_REPO_DIR/.openshift:$OPENSHIFT_REPO_DIR/wsgi:$OPENSHIFT_CELERY_DIR/conf.d:$OPENSHIFT_REPO_DIR:$PYTHONPATH - `echo $OPENSHIFT_CELERY_DIR`usr/celeryd-multi start --cmd=celeryd --pidfile=`echo $OPENSHIFT_CELERY_DIR`etc/celeryd.pid --hostname=$OPENSHIFT_INTERNAL_IP --loglevel=DEBUG `echo $OPENSHIFT_TMP_DIR`celery_log.txt + `echo $OPENSHIFT_CELERY_DIR`usr/celeryd-multi start --cmd=`echo $OPENSHIFT_CELERY_DIR`usr/celeryd --pidfile=`echo $OPENSHIFT_CELERY_DIR`etc/celeryd.pid --hostname=$OPENSHIFT_INTERNAL_IP --loglevel=DEBUG `echo $OPENSHIFT_TMP_DIR`celery_log.txt } function stop { diff --git a/env/OPENSHIFT_CELERY_IMPORTS.erb b/env/OPENSHIFT_CELERY_IMPORTS.erb index c9ead1e..e69de29 100644 --- a/env/OPENSHIFT_CELERY_IMPORTS.erb +++ b/env/OPENSHIFT_CELERY_IMPORTS.erb @@ -1 +0,0 @@ -celerytks.tasks From a4cc138799b9a81456fa2c0704e2c2b94ccd9375 Mon Sep 17 00:00:00 2001 From: zzpwelkin Date: Wed, 1 Jan 2014 13:00:48 +0800 Subject: [PATCH 36/85] [ref] reactor import modules --- conf.d/celeryconfig.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/conf.d/celeryconfig.py b/conf.d/celeryconfig.py index 1c0768c..8f6a138 100644 --- a/conf.d/celeryconfig.py +++ b/conf.d/celeryconfig.py @@ -6,5 +6,6 @@ celery_imports = os.environ.get('OPENSHIFT_CELERY_IMPORTS') -CELERY_IMPORTS = celery_imports.split(',') +if celery_imports: + CELERY_IMPORTS = celery_imports.split(',') From e6567184a16312be5e7975ae6e614a5dc136a14a Mon Sep 17 00:00:00 2001 From: zzpwelkin Date: Mon, 10 Feb 2014 14:02:54 +0800 Subject: [PATCH 37/85] [fix] fixed cmd with logfile tag --- bin/control | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/control b/bin/control index 74faca5..c2e555b 100755 --- a/bin/control +++ b/bin/control @@ -6,7 +6,7 @@ PATH=/bin/:/usr/bin:$PATH function start { export PYTHONPATH=$OPENSHIFT_REPO_DIR/.openshift:$OPENSHIFT_REPO_DIR/wsgi:$OPENSHIFT_CELERY_DIR/conf.d:$OPENSHIFT_REPO_DIR:$PYTHONPATH - `echo $OPENSHIFT_CELERY_DIR`usr/celeryd-multi start --cmd=`echo $OPENSHIFT_CELERY_DIR`usr/celeryd --pidfile=`echo $OPENSHIFT_CELERY_DIR`etc/celeryd.pid --hostname=$OPENSHIFT_INTERNAL_IP --loglevel=DEBUG `echo $OPENSHIFT_TMP_DIR`celery_log.txt + `echo $OPENSHIFT_CELERY_DIR`usr/celeryd-multi start --cmd=`echo $OPENSHIFT_CELERY_DIR`usr/celeryd --pidfile=`echo $OPENSHIFT_CELERY_DIR`etc/celeryd.pid --hostname=$OPENSHIFT_INTERNAL_IP --loglevel=DEBUG --logfile=`echo $OPENSHIFT_TMP_DIR`celery_log.txt } function stop { From 5e1da220779421e823ae00c0b392d351e47fcad2 Mon Sep 17 00:00:00 2001 From: Matti HEIKKILA Date: Fri, 23 May 2014 13:33:42 +0200 Subject: [PATCH 38/85] customized some stuff - hopefully it works --- bin/control | 2 +- bin/setup | 5 +++-- conf.d/celeryconfig.py | 39 +++++++++++++++++++++++++++++++++------ metadata/manifest.yml | 10 +++++++--- usr/celery | 4 ++-- usr/celerybeat | 4 ++-- usr/celeryctl | 4 ++-- usr/celeryd-multi | 4 ++-- usr/celeryev | 4 ++-- 9 files changed, 54 insertions(+), 22 deletions(-) diff --git a/bin/control b/bin/control index c2e555b..46cc894 100755 --- a/bin/control +++ b/bin/control @@ -12,7 +12,7 @@ function start { function stop { if ps -p `cat etc/celeryd.pid` > /dev/null; then - kill -9 `cat etc/celeryd.pid` + kill `cat etc/celeryd.pid` else echo "nothing to kill" fi diff --git a/bin/setup b/bin/setup index 2b8d9f4..f3f14d2 100755 --- a/bin/setup +++ b/bin/setup @@ -1,2 +1,3 @@ -pip install celery==3.0.24 -pip install pymongo +pip install celery==3.1.11 +pip install beautifulsoup4 +pip install requests diff --git a/conf.d/celeryconfig.py b/conf.d/celeryconfig.py index 8f6a138..f9f958c 100644 --- a/conf.d/celeryconfig.py +++ b/conf.d/celeryconfig.py @@ -1,11 +1,38 @@ import os -BROKER_URL = os.environ.get('OPENSHIFT_MONGODB_DB_URL') -CELERY_RESULT_BACKEND = "mongodb" -CELERY_RESULT_DBURI = os.environ.get('OPENSHIFT_MONGODB_DB_URL') +#STORAGE AUTH +STORENAME=os.environ.get("STORAGE_USER") +STOREPASSWORD=os.environ.get("STORAGE_PASSWORD") -celery_imports = os.environ.get('OPENSHIFT_CELERY_IMPORTS') +USERNAME=os.environ.get("BROKER_USER") +PASSWORD=os.environ.get("BROKER_PASSWORD") +BROKER_HOST=os.environ.get("BROKER_HOST") +BROKER_PORT=os.environ.get("BROKER_PORT") +BROKER_VHOST=os.environ.get("BROKER_VHOST") +BROKER_PROTOCOL=os.environ.get("BROKER_PROTOCOL") +BROKER_URL = "%s://%s:%s@%s%s%s" %(BROKER_PROTOCOL, USERNAME, PASSWORD, HOST,PORT,VHOST) + +BROKER_USE_SSL = os.environ.get("BROKER_USE_SSL") + +#RESULT +BROKER_RESULT_PROTOCOL = os.environ.get("BROKER_RESULT_PROTOCOL") +CELERY_RESULT_BACKEND = "%s://%s:%s@%s%s%s" %(BROKER_RESULT_PROTOCOL, USERNAME, PASSWORD, BROKER_HOST,BROKER_PORT,BROKER_VHOST) +CELERY_RESULT_EXCHANGE = os.environ.get("CELERY_RESULT_EXCHANGE") +CELERY_RESULT_EXCHANGE_TYPE = os.environ.get("CELERY_RESULT_EXCHANGE_TYPE") +CELERY_RESULT_PERSISTENT = os.environ.get("CELERY_RESULT_PERSISTENT") +CELERY_RESULT_SERIALIZER = os.environ.get("CELERY_RESULT_SERIALIZER") +CELERY_TASK_RESULT_EXPIRES = os.environ.get("CELERY_TASK_RESULT_EXPIRES") +#TASK + +CELERY_TASK_SERIALIZER = os.environ.get("CELERY_TASK_SERIALIZER") +CELERY_ACCEPT_CONTENT=os.environ.get("CELERY_ACCEPT_CONTENT").split(',') +CELERY_IMPORTS = os.environ.get('OPENSHIFT_CELERY_IMPORTS').split(',') + +#CELERY ROUTES WILL BE DONE AT THE APP UPDATE LEVEL BECAUSE IT CAN EASILY CHANGE / BE AMMENDED +#CELERY_ROUTES = {'endpoint.tasks.process_house': {'queue': 'house_cruncher'}} + +#MISC +CELERY_TIMEZONE = "Europe/Luxembourg" +CELERY_ENABLE_UTC = True -if celery_imports: - CELERY_IMPORTS = celery_imports.split(',') diff --git a/metadata/manifest.yml b/metadata/manifest.yml index 404d2c5..a584a23 100644 --- a/metadata/manifest.yml +++ b/metadata/manifest.yml @@ -4,13 +4,17 @@ Display-Name: Celery Cartridge Description: A cartridge for running the celeryd daemon to allow asynchronous tasks in your application. Version: '0.1' License: ASL 2.0 -Vendor: zzpwelkin +Requires: + - python-2.7 +Vendor: Matti HEIKKILA Cartridge-Version: - - '1.0.1' + - '1.0.2' Cartridge-Vendor: customcarts -Source-Url: https://github.com/zzpwelkin/openshift-celery-cartridge.git +Source-Url: https://bitbucket.org/heico/openshift-celery-cartridge.git Categories: - service +Requires: + - python-2.7 Provides: - celery Cart-Data: diff --git a/usr/celery b/usr/celery index 10ec98c..734e2a2 100755 --- a/usr/celery +++ b/usr/celery @@ -3,11 +3,11 @@ import os; activate_this=os.path.join(os.path.dirname('%svirtenv/bin/' % os.getenv('OPENSHIFT_PYTHON_DIR')), 'activate_this.py'); execfile(activate_this, dict(__file__=activate_this)); del os, activate_this # EASY-INSTALL-ENTRY-SCRIPT: 'celery==3.0.24','console_scripts','celery' -__requires__ = 'celery==3.0.24' +__requires__ = 'celery==3.1.11' import sys from pkg_resources import load_entry_point if __name__ == '__main__': sys.exit( - load_entry_point('celery==3.0.24', 'console_scripts', 'celery')() + load_entry_point('celery==3.1.11', 'console_scripts', 'celery')() ) diff --git a/usr/celerybeat b/usr/celerybeat index aeafaf0..8398c16 100755 --- a/usr/celerybeat +++ b/usr/celerybeat @@ -1,9 +1,9 @@ #!/home/tres/Code/Scribble/screnv/bin/python # EASY-INSTALL-ENTRY-SCRIPT: 'celery==3.0.21','console_scripts','celerybeat' -__requires__ = 'celery==3.0.21' +__requires__ = 'celery==3.1.11' import sys from pkg_resources import load_entry_point sys.exit( - load_entry_point('celery==3.0.21', 'console_scripts', 'celerybeat')() + load_entry_point('celery==3.1.11', 'console_scripts', 'celerybeat')() ) diff --git a/usr/celeryctl b/usr/celeryctl index 703c735..598802b 100755 --- a/usr/celeryctl +++ b/usr/celeryctl @@ -1,9 +1,9 @@ #!/home/tres/Code/Scribble/screnv/bin/python # EASY-INSTALL-ENTRY-SCRIPT: 'celery==3.0.24','console_scripts','celeryctl' -__requires__ = 'celery==3.0.24' +__requires__ = 'celery==3.1.11' import sys from pkg_resources import load_entry_point sys.exit( - load_entry_point('celery==3.0.24', 'console_scripts', 'celeryctl')() + load_entry_point('celery==3.1.11', 'console_scripts', 'celeryctl')() ) diff --git a/usr/celeryd-multi b/usr/celeryd-multi index 12adf31..09e7409 100755 --- a/usr/celeryd-multi +++ b/usr/celeryd-multi @@ -3,10 +3,10 @@ import os; activate_this=os.path.join(os.path.dirname('%svirtenv/bin/' % os.getenv('OPENSHIFT_PYTHON_DIR')), 'activate_this.py'); execfile(activate_this, dict(__file__=activate_this)); del os, activate_this # EASY-INSTALL-ENTRY-SCRIPT: 'celery==3.0.24','console_scripts','celeryd-multi' -__requires__ = 'celery==3.0.24' +__requires__ = 'celery==3.1.11' import sys from pkg_resources import load_entry_point sys.exit( - load_entry_point('celery==3.0.24', 'console_scripts', 'celeryd-multi')() + load_entry_point('celery==3.1.11', 'console_scripts', 'celeryd-multi')() ) diff --git a/usr/celeryev b/usr/celeryev index ac9af3a..5606772 100755 --- a/usr/celeryev +++ b/usr/celeryev @@ -1,9 +1,9 @@ #!/home/tres/Code/Scribble/screnv/bin/python # EASY-INSTALL-ENTRY-SCRIPT: 'celery==3.0.21','console_scripts','celeryev' -__requires__ = 'celery==3.0.21' +__requires__ = 'celery==3.1.11' import sys from pkg_resources import load_entry_point sys.exit( - load_entry_point('celery==3.0.21', 'console_scripts', 'celeryev')() + load_entry_point('celery==3.1.11', 'console_scripts', 'celeryev')() ) From c714989f60874cec4b0cd1a6be6c65a16790f96b Mon Sep 17 00:00:00 2001 From: Matti HEIKKILA Date: Fri, 23 May 2014 13:35:33 +0200 Subject: [PATCH 39/85] modified manifest --- metadata/manifest.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metadata/manifest.yml b/metadata/manifest.yml index a584a23..a91309f 100644 --- a/metadata/manifest.yml +++ b/metadata/manifest.yml @@ -10,7 +10,7 @@ Vendor: Matti HEIKKILA Cartridge-Version: - '1.0.2' Cartridge-Vendor: customcarts -Source-Url: https://bitbucket.org/heico/openshift-celery-cartridge.git +Source-Url: https://github.com/cozezien/openshift-celery-cartridge.git Categories: - service Requires: From 3d5a5e0486e16cc0937c4dc9b377c35f8785e955 Mon Sep 17 00:00:00 2001 From: Matti HEIKKILA Date: Fri, 23 May 2014 14:56:01 +0200 Subject: [PATCH 40/85] changed the way that control started --- bin/control | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/control b/bin/control index 46cc894..0016086 100755 --- a/bin/control +++ b/bin/control @@ -6,7 +6,7 @@ PATH=/bin/:/usr/bin:$PATH function start { export PYTHONPATH=$OPENSHIFT_REPO_DIR/.openshift:$OPENSHIFT_REPO_DIR/wsgi:$OPENSHIFT_CELERY_DIR/conf.d:$OPENSHIFT_REPO_DIR:$PYTHONPATH - `echo $OPENSHIFT_CELERY_DIR`usr/celeryd-multi start --cmd=`echo $OPENSHIFT_CELERY_DIR`usr/celeryd --pidfile=`echo $OPENSHIFT_CELERY_DIR`etc/celeryd.pid --hostname=$OPENSHIFT_INTERNAL_IP --loglevel=DEBUG --logfile=`echo $OPENSHIFT_TMP_DIR`celery_log.txt + `echo $OPENSHIFT_CELERY_DIR`usr/celery worker start --pidfile=`echo $OPENSHIFT_CELERY_DIR`etc/celeryd.pid --loglevel=DEBUG --logfile=`echo $OPENSHIFT_TMP_DIR`celery_log.txt } function stop { From d9499782b3d3a3896cac808c1f1eacd7f8863be9 Mon Sep 17 00:00:00 2001 From: Matti HEIKKILA Date: Fri, 23 May 2014 15:10:34 +0200 Subject: [PATCH 41/85] removed celeryconfig since it just causes problems --- conf.d/celeryconfig.py | 38 -------------------------------------- 1 file changed, 38 deletions(-) delete mode 100644 conf.d/celeryconfig.py diff --git a/conf.d/celeryconfig.py b/conf.d/celeryconfig.py deleted file mode 100644 index f9f958c..0000000 --- a/conf.d/celeryconfig.py +++ /dev/null @@ -1,38 +0,0 @@ -import os - -#STORAGE AUTH -STORENAME=os.environ.get("STORAGE_USER") -STOREPASSWORD=os.environ.get("STORAGE_PASSWORD") - -USERNAME=os.environ.get("BROKER_USER") -PASSWORD=os.environ.get("BROKER_PASSWORD") -BROKER_HOST=os.environ.get("BROKER_HOST") -BROKER_PORT=os.environ.get("BROKER_PORT") -BROKER_VHOST=os.environ.get("BROKER_VHOST") -BROKER_PROTOCOL=os.environ.get("BROKER_PROTOCOL") -BROKER_URL = "%s://%s:%s@%s%s%s" %(BROKER_PROTOCOL, USERNAME, PASSWORD, HOST,PORT,VHOST) - -BROKER_USE_SSL = os.environ.get("BROKER_USE_SSL") - -#RESULT -BROKER_RESULT_PROTOCOL = os.environ.get("BROKER_RESULT_PROTOCOL") -CELERY_RESULT_BACKEND = "%s://%s:%s@%s%s%s" %(BROKER_RESULT_PROTOCOL, USERNAME, PASSWORD, BROKER_HOST,BROKER_PORT,BROKER_VHOST) -CELERY_RESULT_EXCHANGE = os.environ.get("CELERY_RESULT_EXCHANGE") -CELERY_RESULT_EXCHANGE_TYPE = os.environ.get("CELERY_RESULT_EXCHANGE_TYPE") -CELERY_RESULT_PERSISTENT = os.environ.get("CELERY_RESULT_PERSISTENT") -CELERY_RESULT_SERIALIZER = os.environ.get("CELERY_RESULT_SERIALIZER") -CELERY_TASK_RESULT_EXPIRES = os.environ.get("CELERY_TASK_RESULT_EXPIRES") -#TASK - -CELERY_TASK_SERIALIZER = os.environ.get("CELERY_TASK_SERIALIZER") -CELERY_ACCEPT_CONTENT=os.environ.get("CELERY_ACCEPT_CONTENT").split(',') -CELERY_IMPORTS = os.environ.get('OPENSHIFT_CELERY_IMPORTS').split(',') - -#CELERY ROUTES WILL BE DONE AT THE APP UPDATE LEVEL BECAUSE IT CAN EASILY CHANGE / BE AMMENDED -#CELERY_ROUTES = {'endpoint.tasks.process_house': {'queue': 'house_cruncher'}} - -#MISC -CELERY_TIMEZONE = "Europe/Luxembourg" -CELERY_ENABLE_UTC = True - - From 80778ec788d3e5a2079c9b91e36c0036776359fd Mon Sep 17 00:00:00 2001 From: Matti HEIKKILA Date: Fri, 23 May 2014 15:14:20 +0200 Subject: [PATCH 42/85] trying to get this working --- bin/control | 4 ++-- usr/celeryd | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/bin/control b/bin/control index 0016086..078f7a3 100755 --- a/bin/control +++ b/bin/control @@ -5,8 +5,8 @@ PATH=/bin/:/usr/bin:$PATH #source $OPENSHIFT_CARTRIDGE_SDK_BASH function start { - export PYTHONPATH=$OPENSHIFT_REPO_DIR/.openshift:$OPENSHIFT_REPO_DIR/wsgi:$OPENSHIFT_CELERY_DIR/conf.d:$OPENSHIFT_REPO_DIR:$PYTHONPATH - `echo $OPENSHIFT_CELERY_DIR`usr/celery worker start --pidfile=`echo $OPENSHIFT_CELERY_DIR`etc/celeryd.pid --loglevel=DEBUG --logfile=`echo $OPENSHIFT_TMP_DIR`celery_log.txt + export PYTHONPATH=$OPENSHIFT_REPO_DIR/.openshift:$OPENSHIFT_REPO_DIR/wsgi:$OPENSHIFT_REPO_DIR:$PYTHONPATH + `echo $OPENSHIFT_CELERY_DIR`usr/celery worker start --pidfile=`echo $OPENSHIFT_CELERY_DIR`etc/celeryd.pid --loglevel=DEBUG --logfile=`echo $OPENSHIFT_TMP_DIR`celery_log.txt } function stop { diff --git a/usr/celeryd b/usr/celeryd index af87eb7..5ebe85a 100755 --- a/usr/celeryd +++ b/usr/celeryd @@ -3,11 +3,11 @@ import os; activate_this=os.path.join(os.path.dirname('%svirtenv/bin/' % os.getenv('OPENSHIFT_PYTHON_DIR')), 'activate_this.py'); execfile(activate_this, dict(__file__=activate_this)); del os, activate_this # EASY-INSTALL-ENTRY-SCRIPT: 'celery==3.0.24','console_scripts','celeryd' -__requires__ = 'celery==3.0.24' +__requires__ = 'celery==3.1.11' import sys from pkg_resources import load_entry_point if __name__ == '__main__': sys.exit( - load_entry_point('celery==3.0.24', 'console_scripts', 'celeryd')() + load_entry_point('celery==3.1.11', 'console_scripts', 'celeryd')() ) From 84569ddf8d4f476b073db26a4237d07287459ca9 Mon Sep 17 00:00:00 2001 From: Matti HEIKKILA Date: Fri, 23 May 2014 15:16:57 +0200 Subject: [PATCH 43/85] blarg --- bin/control | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/control b/bin/control index 078f7a3..b67431d 100755 --- a/bin/control +++ b/bin/control @@ -6,7 +6,7 @@ PATH=/bin/:/usr/bin:$PATH function start { export PYTHONPATH=$OPENSHIFT_REPO_DIR/.openshift:$OPENSHIFT_REPO_DIR/wsgi:$OPENSHIFT_REPO_DIR:$PYTHONPATH - `echo $OPENSHIFT_CELERY_DIR`usr/celery worker start --pidfile=`echo $OPENSHIFT_CELERY_DIR`etc/celeryd.pid --loglevel=DEBUG --logfile=`echo $OPENSHIFT_TMP_DIR`celery_log.txt + echo "`echo $OPENSHIFT_CELERY_DIR`usr/celery worker start --pidfile=`echo $OPENSHIFT_CELERY_DIR`etc/celeryd.pid --loglevel=DEBUG --logfile=`echo $OPENSHIFT_TMP_DIR`celery_log.txt" >> `echo $OPENSHIFT_TMP_DIR`blarg.txt } function stop { From 1c525bd835192cc7ce619b569ece81bf197f9011 Mon Sep 17 00:00:00 2001 From: Matti HEIKKILA Date: Fri, 23 May 2014 17:05:33 +0200 Subject: [PATCH 44/85] removed some stuff --- bin/control | 8 ++- bin/mock_server.rb | 14 ---- bin/setup | 2 - conf.d/celeryconfig.py.erb.bg | 65 ------------------- env/OPENSHIFT_CELERY_BROKER_TRANS.erb | 1 - env/OPENSHIFT_CELERY_BROKER_URL.erb | 1 - env/OPENSHIFT_CELERY_IMPORTS.erb | 0 env/OPENSHIFT_CELERY_RESULT_BACKEND.erb | 1 - template.git/.openshift/action_hooks/build | 1 - template.git/.openshift/action_hooks/deploy | 6 -- .../.openshift/action_hooks/post-deploy | 1 - .../.openshift/action_hooks/post_deploy | 4 -- .../.openshift/action_hooks/pre-build | 1 - template.git/celerytks/__init__.py | 0 template.git/celerytks/tasks.py | 13 ---- template/celerytks/tasks.py | 1 + usr/celeryd | 13 ---- usr/celeryd-multi | 12 ---- usr/scrapy.py | 19 ------ 19 files changed, 6 insertions(+), 157 deletions(-) delete mode 100755 bin/mock_server.rb delete mode 100644 conf.d/celeryconfig.py.erb.bg delete mode 100644 env/OPENSHIFT_CELERY_BROKER_TRANS.erb delete mode 100644 env/OPENSHIFT_CELERY_BROKER_URL.erb delete mode 100644 env/OPENSHIFT_CELERY_IMPORTS.erb delete mode 100644 env/OPENSHIFT_CELERY_RESULT_BACKEND.erb delete mode 100644 template.git/.openshift/action_hooks/build delete mode 100755 template.git/.openshift/action_hooks/deploy delete mode 100644 template.git/.openshift/action_hooks/post-deploy delete mode 100755 template.git/.openshift/action_hooks/post_deploy delete mode 100644 template.git/.openshift/action_hooks/pre-build delete mode 100644 template.git/celerytks/__init__.py delete mode 100644 template.git/celerytks/tasks.py delete mode 100755 usr/celeryd delete mode 100755 usr/celeryd-multi delete mode 100644 usr/scrapy.py diff --git a/bin/control b/bin/control index b67431d..73e6dc4 100755 --- a/bin/control +++ b/bin/control @@ -6,13 +6,14 @@ PATH=/bin/:/usr/bin:$PATH function start { export PYTHONPATH=$OPENSHIFT_REPO_DIR/.openshift:$OPENSHIFT_REPO_DIR/wsgi:$OPENSHIFT_REPO_DIR:$PYTHONPATH - echo "`echo $OPENSHIFT_CELERY_DIR`usr/celery worker start --pidfile=`echo $OPENSHIFT_CELERY_DIR`etc/celeryd.pid --loglevel=DEBUG --logfile=`echo $OPENSHIFT_TMP_DIR`celery_log.txt" >> `echo $OPENSHIFT_TMP_DIR`blarg.txt + echo "cartridge ready for code" +# echo "`echo $OPENSHIFT_CELERY_DIR`usr/celery worker start --pidfile=`echo $OPENSHIFT_CELERY_DIR`etc/celeryd.pid --loglevel=DEBUG --logfile=`echo $OPENSHIFT_TMP_DIR`celery_log.txt" >> `echo $OPENSHIFT_TMP_DIR`blarg.txt } function stop { - if ps -p `cat etc/celeryd.pid` > /dev/null; + if ps -p `cat ${OPENSHIFT_CELERY_DIR}etc/celeryd.pid` > /dev/null; then - kill `cat etc/celeryd.pid` + kill `cat ${OPENSHIFT_CELERY_DIR}etc/celeryd.pid` else echo "nothing to kill" fi @@ -21,6 +22,7 @@ function stop { function restart { echo "restarted" + } function catchall { diff --git a/bin/mock_server.rb b/bin/mock_server.rb deleted file mode 100755 index e9ef115..0000000 --- a/bin/mock_server.rb +++ /dev/null @@ -1,14 +0,0 @@ -#!/usr/bin/env ruby -require 'webrick' -include WEBrick - -config = {} -config.update(:Port => 8080) -config.update(:BindAddress => ARGV[0]) -config.update(:DocumentRoot => ARGV[1]) -server = HTTPServer.new(config) -['INT', 'TERM'].each {|signal| - trap(signal) {server.shutdown} -} - -server.start diff --git a/bin/setup b/bin/setup index f3f14d2..eb2ab22 100755 --- a/bin/setup +++ b/bin/setup @@ -1,3 +1 @@ pip install celery==3.1.11 -pip install beautifulsoup4 -pip install requests diff --git a/conf.d/celeryconfig.py.erb.bg b/conf.d/celeryconfig.py.erb.bg deleted file mode 100644 index 73b1374..0000000 --- a/conf.d/celeryconfig.py.erb.bg +++ /dev/null @@ -1,65 +0,0 @@ -print 'Configuring celeryd with default cartridge config file' -<% -broker_type = ENV.fetch('OPENSHIFT_CELERY_BROKER_TRANS', 'mongodb') -if broker_type == "mongodb" %> -## Broker settings. -##default_url = 'mongodb://localshot:12607' -BROKER_URL = '<%= ENV.fetch('OPENSHIFT_CELERY_BROKER_URL', 'mongodb://localshot:12607') %>' -## Using the database to store task state and results. -CELERY_RESULT_BACKEND = "mongodb" -CELERY_RESULT_DBURI = '<%= ENV.fetch('OPENSHIFT_CELERY_RESULT_BACKEND', 'mongodb://localshot:12607') %>' - -<% elsif broker_type == "amqp" %> -## Broker settings -BROKER_URL = 'amqp://<%= ENV.fetch('OPENSHIFT_CELERY_BROKER_URL', 'guest:guest@localhost:5672//') %>' -<% elsif broker_type == "redis" %> -## Broker settings -BROKER_URL = 'redis://<%= ENV.fetch('OPENSHIFT_CELERY_BROKER_URL', 'localhost:6379/0') %>' -BROKER_TRANSPORT_OPTIONS = {'visibility_timeout': <%= ENV.fetch('OPENSHIFT_CELERY_VISIBILITY_TIME', 3600) %> } -CELERY_RESULT_BACKEND = 'redis://<%= ENV.fetch('OPENSHIFT_CELERY_RESULT_BACKEND', 'localhost:6379/0') %>' -<% end %> -# List of modules to import when celery starts. -<% - celery_imports = ENV.fetch('OPENSHIFT_CELERY_IMPORTS', '').split(',') - celery_imports = celery_imports.map { |a| "'" + a + "'" } - celery_imports = celery_imports.join(",") -%> -CELERY_IMPORTS = (<%= celery_imports %>, ) - -CELERYD_LOG_FILE = '<%= ENV.fetch('OPENSHIFT_TMP_DIR') %>log/celery/worker1.log' - -""" -BROKER_BACKEND = 'mongodb' -CELERY_RESULT_BACKEND = 'mongodb' -BROKER_HOST = os.environ.get('OPENSHIFT_MONGODB_DB_HOST') -BROKER_PORT = os.environ.get('OPENSHIFT_MONGODB_DB_PORT') -BROKER_USER = os.environ.get('OPENSHIFT_MONGODB_DB_USERNAME') -BROKER_PASSWORD = os.environ.get('OPENSHIFT_MONGODB_DB_PASSWORD') -BROKER_VHOST = 'celery_tasks' -BROKER_URL = 'mongodb://admin:XCf67tq_BqKU@writeown-tresback.rhcloud.com/celery_tasks' -CELERY_RESULT_BACKEND = 'mongodb://admin:XCf67tq_BqKU@writeown-tresback.rhcloud.com/celery_tasks' -BROKER_URL = 'mongodb://admin:XCf67tq_BqKU@127.8.167.130/celery_tasks' -#"%scelery_tasks" % os.environ.get('OPENSHIFT_MONGODB_DB_URL') -BROKER_BACKEND = 'mongodb' -CELERY_RESULT_BACKEND = 'mongodb' -BROKER_HOST = os.environ.get('OPENSHIFT_MONGODB_DB_HOST') -BROKER_PORT = os.environ.get('OPENSHIFT_MONGODB_DB_PORT') -BROKER_TRANSPORT = 'mongodb' -BROKER_VHOST = 'celery_tasks' -CELERY_MONGODB_BACKEND_SETTINGS = { - "host": os.environ.get('OPENSHIFT_MONGODB_DB_HOST'), - "port": os.environ.get('OPENSHIFT_MONGODB_DB_PORT'), - "database": 'celery_tasks', -} -CELERY_MONGODB_BACKEND_SETTINGS = { - "host": os.environ.get('OPENSHIFT_MONGODB_DB_HOST'), - "port": os.environ.get('OPENSHIFT_MONGODB_DB_PORT'), - "database": 'celery_tasks', - "user": os.environ.get('OPENSHIFT_MONGODB_DB_USERNAME'), - "password": os.environ.get('OPENSHIFT_MONGODB_DB_PASSWORD') -} - -BROKER_BACKEND = 'redis' -BROKER_URL = "redis://127.8.167.131:16379/0" -CELERY_RESULT_BACKEND = "redis://127.8.167.131:16379/0" -""" diff --git a/env/OPENSHIFT_CELERY_BROKER_TRANS.erb b/env/OPENSHIFT_CELERY_BROKER_TRANS.erb deleted file mode 100644 index 97ee1da..0000000 --- a/env/OPENSHIFT_CELERY_BROKER_TRANS.erb +++ /dev/null @@ -1 +0,0 @@ -mongodb diff --git a/env/OPENSHIFT_CELERY_BROKER_URL.erb b/env/OPENSHIFT_CELERY_BROKER_URL.erb deleted file mode 100644 index 33ab847..0000000 --- a/env/OPENSHIFT_CELERY_BROKER_URL.erb +++ /dev/null @@ -1 +0,0 @@ -<%= ENV['OPENSHIFT_MONGODB_DB_URL'] %> diff --git a/env/OPENSHIFT_CELERY_IMPORTS.erb b/env/OPENSHIFT_CELERY_IMPORTS.erb deleted file mode 100644 index e69de29..0000000 diff --git a/env/OPENSHIFT_CELERY_RESULT_BACKEND.erb b/env/OPENSHIFT_CELERY_RESULT_BACKEND.erb deleted file mode 100644 index 33ab847..0000000 --- a/env/OPENSHIFT_CELERY_RESULT_BACKEND.erb +++ /dev/null @@ -1 +0,0 @@ -<%= ENV['OPENSHIFT_MONGODB_DB_URL'] %> diff --git a/template.git/.openshift/action_hooks/build b/template.git/.openshift/action_hooks/build deleted file mode 100644 index 8b13789..0000000 --- a/template.git/.openshift/action_hooks/build +++ /dev/null @@ -1 +0,0 @@ - diff --git a/template.git/.openshift/action_hooks/deploy b/template.git/.openshift/action_hooks/deploy deleted file mode 100755 index 3875983..0000000 --- a/template.git/.openshift/action_hooks/deploy +++ /dev/null @@ -1,6 +0,0 @@ -#!/bin/bash -# This deploy hook gets executed after dependencies are resolved and the -# build hook has been run but before the application has been started back -# up again. This script gets executed directly, so it could be python, php, -# ruby, etc. - diff --git a/template.git/.openshift/action_hooks/post-deploy b/template.git/.openshift/action_hooks/post-deploy deleted file mode 100644 index 8b13789..0000000 --- a/template.git/.openshift/action_hooks/post-deploy +++ /dev/null @@ -1 +0,0 @@ - diff --git a/template.git/.openshift/action_hooks/post_deploy b/template.git/.openshift/action_hooks/post_deploy deleted file mode 100755 index d38c5da..0000000 --- a/template.git/.openshift/action_hooks/post_deploy +++ /dev/null @@ -1,4 +0,0 @@ -#!/bin/bash - -source $OPENSHIFT_HOMEDIR/python/virtenv/bin/activate -export PYTHONPATH=`echo $OPENSHIFT_REPO_DIR`:$PYTHONPATH diff --git a/template.git/.openshift/action_hooks/pre-build b/template.git/.openshift/action_hooks/pre-build deleted file mode 100644 index 8b13789..0000000 --- a/template.git/.openshift/action_hooks/pre-build +++ /dev/null @@ -1 +0,0 @@ - diff --git a/template.git/celerytks/__init__.py b/template.git/celerytks/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/template.git/celerytks/tasks.py b/template.git/celerytks/tasks.py deleted file mode 100644 index 2041943..0000000 --- a/template.git/celerytks/tasks.py +++ /dev/null @@ -1,13 +0,0 @@ -import celery -app = celery.Celery() - -@app.task -def addtest(x, y): - return x + y - -@app.task -def run_spiders(name, **kwargs): - """ - @param name: spider name - """ - pass diff --git a/template/celerytks/tasks.py b/template/celerytks/tasks.py index 9f779ee..ac79360 100644 --- a/template/celerytks/tasks.py +++ b/template/celerytks/tasks.py @@ -1,4 +1,5 @@ import celery + app = celery.Celery() app.config_from_object('celeryconfig') diff --git a/usr/celeryd b/usr/celeryd deleted file mode 100755 index 5ebe85a..0000000 --- a/usr/celeryd +++ /dev/null @@ -1,13 +0,0 @@ -#!/usr/bin/env python2.7 - -import os; activate_this=os.path.join(os.path.dirname('%svirtenv/bin/' % os.getenv('OPENSHIFT_PYTHON_DIR')), 'activate_this.py'); execfile(activate_this, dict(__file__=activate_this)); del os, activate_this - -# EASY-INSTALL-ENTRY-SCRIPT: 'celery==3.0.24','console_scripts','celeryd' -__requires__ = 'celery==3.1.11' -import sys -from pkg_resources import load_entry_point - -if __name__ == '__main__': - sys.exit( - load_entry_point('celery==3.1.11', 'console_scripts', 'celeryd')() - ) diff --git a/usr/celeryd-multi b/usr/celeryd-multi deleted file mode 100755 index 09e7409..0000000 --- a/usr/celeryd-multi +++ /dev/null @@ -1,12 +0,0 @@ -#!/usr/bin/env python2.7 - -import os; activate_this=os.path.join(os.path.dirname('%svirtenv/bin/' % os.getenv('OPENSHIFT_PYTHON_DIR')), 'activate_this.py'); execfile(activate_this, dict(__file__=activate_this)); del os, activate_this - -# EASY-INSTALL-ENTRY-SCRIPT: 'celery==3.0.24','console_scripts','celeryd-multi' -__requires__ = 'celery==3.1.11' -import sys -from pkg_resources import load_entry_point - -sys.exit( - load_entry_point('celery==3.1.11', 'console_scripts', 'celeryd-multi')() -) diff --git a/usr/scrapy.py b/usr/scrapy.py deleted file mode 100644 index 1710a64..0000000 --- a/usr/scrapy.py +++ /dev/null @@ -1,19 +0,0 @@ -from scrapy.settings import CrawlerSettings -from scrapy.crawler import CrawlerProcess - -spider_example_name = 'news_spider' - -appname = 'openshift' - -os.environ.setdefault('SCRAPER_SETTINGS_MODULE', '{0}.scraper.settings'.format(appname)) -scrapy_settings_model = __import__(os.getenv('SCRAPER_SETTINGS_MODULE'), fromlist=['']) - - -settings = CrawlerSettings(scrapy_settings_model) -prs = CrawlerProcess(settings) -crawler = prs.create_crawler() -for name, spd in crawler.spiders._spiders.iteritems(): - if name == spider_example_name: - spidercls = spd -crawler.crawl(spidercls(id=1)) -prs.start() From d06afab478e6652580224e7b611210b67111be13 Mon Sep 17 00:00:00 2001 From: Matti HEIKKILA Date: Fri, 23 May 2014 17:22:35 +0200 Subject: [PATCH 45/85] test --- metadata/manifest.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/metadata/manifest.yml b/metadata/manifest.yml index a91309f..eac3567 100644 --- a/metadata/manifest.yml +++ b/metadata/manifest.yml @@ -13,8 +13,8 @@ Cartridge-Vendor: customcarts Source-Url: https://github.com/cozezien/openshift-celery-cartridge.git Categories: - service -Requires: - - python-2.7 + - web_framework +Requires: python = 2.7 Provides: - celery Cart-Data: From b0fd137adcc0613608f4dccc479880de48a2f221 Mon Sep 17 00:00:00 2001 From: Matti HEIKKILA Date: Fri, 23 May 2014 17:23:44 +0200 Subject: [PATCH 46/85] blarg --- metadata/manifest.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metadata/manifest.yml b/metadata/manifest.yml index eac3567..6ff14e5 100644 --- a/metadata/manifest.yml +++ b/metadata/manifest.yml @@ -14,7 +14,7 @@ Source-Url: https://github.com/cozezien/openshift-celery-cartridge.git Categories: - service - web_framework -Requires: python = 2.7 +Requires: python-2.7 Provides: - celery Cart-Data: From 198f0e05fcdb5e323d68776af105b6ef1ffb40f2 Mon Sep 17 00:00:00 2001 From: Matti HEIKKILA Date: Mon, 26 May 2014 09:45:23 +0200 Subject: [PATCH 47/85] remove python 2.7 and see what happens --- metadata/manifest.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/metadata/manifest.yml b/metadata/manifest.yml index 6ff14e5..f5950bd 100644 --- a/metadata/manifest.yml +++ b/metadata/manifest.yml @@ -4,8 +4,6 @@ Display-Name: Celery Cartridge Description: A cartridge for running the celeryd daemon to allow asynchronous tasks in your application. Version: '0.1' License: ASL 2.0 -Requires: - - python-2.7 Vendor: Matti HEIKKILA Cartridge-Version: - '1.0.2' @@ -14,7 +12,6 @@ Source-Url: https://github.com/cozezien/openshift-celery-cartridge.git Categories: - service - web_framework -Requires: python-2.7 Provides: - celery Cart-Data: From 3d06ac819ea988c87f160399799768336adf3b16 Mon Sep 17 00:00:00 2001 From: Matti HEIKKILA Date: Mon, 26 May 2014 13:27:48 +0200 Subject: [PATCH 48/85] saving the fact we don't need the setup and also requiring python 2.7 --- bin/setup | 1 - metadata/manifest.yml | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) delete mode 100755 bin/setup diff --git a/bin/setup b/bin/setup deleted file mode 100755 index eb2ab22..0000000 --- a/bin/setup +++ /dev/null @@ -1 +0,0 @@ -pip install celery==3.1.11 diff --git a/metadata/manifest.yml b/metadata/manifest.yml index f5950bd..245f249 100644 --- a/metadata/manifest.yml +++ b/metadata/manifest.yml @@ -11,7 +11,7 @@ Cartridge-Vendor: customcarts Source-Url: https://github.com/cozezien/openshift-celery-cartridge.git Categories: - service - - web_framework +Requires: python-2.7 Provides: - celery Cart-Data: From 2b35e3a16895581cbd5924ddb52d1cb00814039f Mon Sep 17 00:00:00 2001 From: edmcdonagh Date: Sat, 13 Dec 2014 00:14:11 +0000 Subject: [PATCH 49/85] Removed celery version numbers --- usr/celery | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/usr/celery b/usr/celery index 734e2a2..ed4d988 100755 --- a/usr/celery +++ b/usr/celery @@ -3,11 +3,11 @@ import os; activate_this=os.path.join(os.path.dirname('%svirtenv/bin/' % os.getenv('OPENSHIFT_PYTHON_DIR')), 'activate_this.py'); execfile(activate_this, dict(__file__=activate_this)); del os, activate_this # EASY-INSTALL-ENTRY-SCRIPT: 'celery==3.0.24','console_scripts','celery' -__requires__ = 'celery==3.1.11' +__requires__ = 'celery' import sys from pkg_resources import load_entry_point if __name__ == '__main__': sys.exit( - load_entry_point('celery==3.1.11', 'console_scripts', 'celery')() + load_entry_point('celery', 'console_scripts', 'celery')() ) From 3bfcc6d565b42eddfa1182557867ca2f2c788c53 Mon Sep 17 00:00:00 2001 From: edmcdonagh Date: Sat, 13 Dec 2014 00:14:34 +0000 Subject: [PATCH 50/85] Removed celery version numbers --- usr/celerybeat | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/usr/celerybeat b/usr/celerybeat index 8398c16..1a672a6 100755 --- a/usr/celerybeat +++ b/usr/celerybeat @@ -5,5 +5,5 @@ import sys from pkg_resources import load_entry_point sys.exit( - load_entry_point('celery==3.1.11', 'console_scripts', 'celerybeat')() + load_entry_point('celery', 'console_scripts', 'celerybeat')() ) From 250387762d8127f9c7c9958842e0857f1fc1f7f4 Mon Sep 17 00:00:00 2001 From: edmcdonagh Date: Sat, 13 Dec 2014 00:14:50 +0000 Subject: [PATCH 51/85] Removed celery version numbers --- usr/celerybeat | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/usr/celerybeat b/usr/celerybeat index 1a672a6..2f7fd10 100755 --- a/usr/celerybeat +++ b/usr/celerybeat @@ -1,6 +1,6 @@ #!/home/tres/Code/Scribble/screnv/bin/python # EASY-INSTALL-ENTRY-SCRIPT: 'celery==3.0.21','console_scripts','celerybeat' -__requires__ = 'celery==3.1.11' +__requires__ = 'celery' import sys from pkg_resources import load_entry_point From 5a00c5cc82fbaa49ced3ec8779d671a0f303e63e Mon Sep 17 00:00:00 2001 From: edmcdonagh Date: Sat, 13 Dec 2014 00:15:15 +0000 Subject: [PATCH 52/85] Removed celery version numbers --- usr/celeryctl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/usr/celeryctl b/usr/celeryctl index 598802b..3ce9912 100755 --- a/usr/celeryctl +++ b/usr/celeryctl @@ -1,9 +1,9 @@ #!/home/tres/Code/Scribble/screnv/bin/python # EASY-INSTALL-ENTRY-SCRIPT: 'celery==3.0.24','console_scripts','celeryctl' -__requires__ = 'celery==3.1.11' +__requires__ = 'celery' import sys from pkg_resources import load_entry_point sys.exit( - load_entry_point('celery==3.1.11', 'console_scripts', 'celeryctl')() + load_entry_point('celery', 'console_scripts', 'celeryctl')() ) From 8953f4a93072895bfaf0a662b20eda557c057051 Mon Sep 17 00:00:00 2001 From: edmcdonagh Date: Sat, 13 Dec 2014 00:15:52 +0000 Subject: [PATCH 53/85] Removed celery version numbers --- usr/celeryev | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/usr/celeryev b/usr/celeryev index 5606772..ead9de0 100755 --- a/usr/celeryev +++ b/usr/celeryev @@ -1,9 +1,9 @@ #!/home/tres/Code/Scribble/screnv/bin/python # EASY-INSTALL-ENTRY-SCRIPT: 'celery==3.0.21','console_scripts','celeryev' -__requires__ = 'celery==3.1.11' +__requires__ = 'celery' import sys from pkg_resources import load_entry_point sys.exit( - load_entry_point('celery==3.1.11', 'console_scripts', 'celeryev')() + load_entry_point('celery', 'console_scripts', 'celeryev')() ) From 0925b9576adaa3224574dec867955e7faceb3a61 Mon Sep 17 00:00:00 2001 From: Is Isilon Date: Fri, 10 Apr 2015 14:33:14 +0800 Subject: [PATCH 54/85] trying to get it working on updated openshift --- bin/control | 23 ++++++++++++++++++----- usr/celery | 7 +++++-- 2 files changed, 23 insertions(+), 7 deletions(-) diff --git a/bin/control b/bin/control index 73e6dc4..76500f1 100755 --- a/bin/control +++ b/bin/control @@ -1,19 +1,19 @@ #!/bin/bash -e -PATH=/bin/:/usr/bin:$PATH - +PATH=${OPENSHIFT_CELERY_DIR}bin/:${OPENSHIFT_CELERY_DIR}usr/bin:$PATH #source $OPENSHIFT_CARTRIDGE_SDK_BASH function start { export PYTHONPATH=$OPENSHIFT_REPO_DIR/.openshift:$OPENSHIFT_REPO_DIR/wsgi:$OPENSHIFT_REPO_DIR:$PYTHONPATH echo "cartridge ready for code" -# echo "`echo $OPENSHIFT_CELERY_DIR`usr/celery worker start --pidfile=`echo $OPENSHIFT_CELERY_DIR`etc/celeryd.pid --loglevel=DEBUG --logfile=`echo $OPENSHIFT_TMP_DIR`celery_log.txt" >> `echo $OPENSHIFT_TMP_DIR`blarg.txt + echo "${OPENSHIFT_CELERY_DIR}usr/celery worker -A ${OPENSHIFT_CELERY_PYTHON_PKG_NAME} -b ${OPENSHIFT_CELERY_BROKER_URL} --pidfile=${OPENSHIFT_CELERY_DIR}etc/celeryd.pid --loglevel=DEBUG --logfile=${OPENSHIFT_TMP_DIR}celery_log.txt" >> `echo $OPENSHIFT_TMP_DIR`blarg.txt + ${OPENSHIFT_CELERY_DIR}usr/celery multi start worker --app=${OPENSHIFT_CELERY_PYTHON_PKG_NAME} --broker=${OPENSHIFT_CELERY_BROKER_URL} --pidfile=${OPENSHIFT_CELERY_DIR}etc/celeryd.pid --hostname=${OPENSHIFT_APP_DNS} --loglevel=DEBUG --logfile=${OPENSHIFT_TMP_DIR}celery_log.txt } function stop { if ps -p `cat ${OPENSHIFT_CELERY_DIR}etc/celeryd.pid` > /dev/null; then - kill `cat ${OPENSHIFT_CELERY_DIR}etc/celeryd.pid` + kill -9 `cat ${OPENSHIFT_CELERY_DIR}etc/celeryd.pid` else echo "nothing to kill" fi @@ -21,10 +21,23 @@ function stop { } function restart { - echo "restarted" + celery multi restart worker --pidfile=${OPENSHIFT_CELERY_DIR}etc/celeryd.pid + echo "not yet implemented" } +function status() { + pid=`cat ${OPENSHIFT_CELERY_DIR}etc/celeryd.pid` + info=`celery multi show worker` + client_result $pwd + if [ -f $pid ] && ( kill -0 $(cat $pid) ); then + client_result "Celery is running ${info}" + client_result " Connect to: $OPENSHIFT_GEAR_DNS:$OPENSHIFT_CELERY_IDENT" + else + client_result "Celery is not running" + fi +} + function catchall { echo "not yet implemented" } diff --git a/usr/celery b/usr/celery index ed4d988..de3cef9 100755 --- a/usr/celery +++ b/usr/celery @@ -1,8 +1,11 @@ #!/usr/bin/env python2.7 -import os; activate_this=os.path.join(os.path.dirname('%svirtenv/bin/' % os.getenv('OPENSHIFT_PYTHON_DIR')), 'activate_this.py'); execfile(activate_this, dict(__file__=activate_this)); del os, activate_this +import os +activate_this=os.path.join(os.path.dirname('%svirtenv/bin/' % os.getenv('OPENSHIFT_PYTHON_DIR')), 'activate_this.py') +execfile(activate_this, dict(__file__=activate_this)) +del os, activate_this -# EASY-INSTALL-ENTRY-SCRIPT: 'celery==3.0.24','console_scripts','celery' +# EASY-INSTALL-ENTRY-SCRIPT: 'celery','console_scripts','celery' __requires__ = 'celery' import sys from pkg_resources import load_entry_point From 04c10f056e89ab9e5d65c12a22fa75c1c7498cf5 Mon Sep 17 00:00:00 2001 From: Is Isilon Date: Fri, 10 Apr 2015 14:41:50 +0800 Subject: [PATCH 55/85] trying to get it working on updated openshift --- README.md | 8 +++----- bin/control | 8 ++++---- 2 files changed, 7 insertions(+), 9 deletions(-) diff --git a/README.md b/README.md index f0b486f..a1bde68 100644 --- a/README.md +++ b/README.md @@ -6,12 +6,10 @@ Cartridge to Expose Celery as a Daemon on OpenShift Environment Variables --------------------- -- OPENSHIFT_CELERY_BROKER_TRANS : Defines the broker type that celery will use. Current available options are: - -- mongodb : DEFAULT - -- amqp - -- redis - OPENSHIFT_CELERY_BROKER_URL : The connection URL for the broker, omitting the transport. For example, an amqp value may look like: guest:guest@localhost:5672// -- + - OPENSHIFT_CELERY_IMPORTS : Defines the modules that celery should import. Currently only supports 1 module import path Currently only supports MongoDB as backend for queues, but in progress to support more environment variables to allow vairous backends and configurations. + +To install rhc cartridge-add https://raw.github.com/wassname/openshift-celery-cartridge/master/metadata/manifest.yml -a diff --git a/bin/control b/bin/control index 76500f1..2736fa3 100755 --- a/bin/control +++ b/bin/control @@ -1,13 +1,14 @@ #!/bin/bash -e PATH=${OPENSHIFT_CELERY_DIR}bin/:${OPENSHIFT_CELERY_DIR}usr/bin:$PATH -#source $OPENSHIFT_CARTRIDGE_SDK_BASH + +#TODO add scaling, better status, variable worker names function start { export PYTHONPATH=$OPENSHIFT_REPO_DIR/.openshift:$OPENSHIFT_REPO_DIR/wsgi:$OPENSHIFT_REPO_DIR:$PYTHONPATH echo "cartridge ready for code" echo "${OPENSHIFT_CELERY_DIR}usr/celery worker -A ${OPENSHIFT_CELERY_PYTHON_PKG_NAME} -b ${OPENSHIFT_CELERY_BROKER_URL} --pidfile=${OPENSHIFT_CELERY_DIR}etc/celeryd.pid --loglevel=DEBUG --logfile=${OPENSHIFT_TMP_DIR}celery_log.txt" >> `echo $OPENSHIFT_TMP_DIR`blarg.txt - ${OPENSHIFT_CELERY_DIR}usr/celery multi start worker --app=${OPENSHIFT_CELERY_PYTHON_PKG_NAME} --broker=${OPENSHIFT_CELERY_BROKER_URL} --pidfile=${OPENSHIFT_CELERY_DIR}etc/celeryd.pid --hostname=${OPENSHIFT_APP_DNS} --loglevel=DEBUG --logfile=${OPENSHIFT_TMP_DIR}celery_log.txt + ${OPENSHIFT_CELERY_DIR}usr/celery multi start worker --app=${OPENSHIFT_CELERY_IMPORTS} --broker=${OPENSHIFT_CELERY_BROKER_URL} --pidfile=${OPENSHIFT_CELERY_DIR}etc/celeryd.pid --hostname=${OPENSHIFT_APP_DNS} --loglevel=DEBUG --logfile=${OPENSHIFT_TMP_DIR}celery_log.txt } function stop { @@ -31,8 +32,7 @@ function status() { info=`celery multi show worker` client_result $pwd if [ -f $pid ] && ( kill -0 $(cat $pid) ); then - client_result "Celery is running ${info}" - client_result " Connect to: $OPENSHIFT_GEAR_DNS:$OPENSHIFT_CELERY_IDENT" + client_result "Celery is running ${info} at pid:${pid}" else client_result "Celery is not running" fi From b8f920a59ea650c660012f7f333bb09f5ad9177c Mon Sep 17 00:00:00 2001 From: Is Isilon Date: Fri, 10 Apr 2015 15:01:15 +0800 Subject: [PATCH 56/85] updated readme --- README.md | 21 +++++++++++++++++---- metadata/manifest.yml | 7 ++++--- 2 files changed, 21 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index a1bde68..661fb82 100644 --- a/README.md +++ b/README.md @@ -6,10 +6,23 @@ Cartridge to Expose Celery as a Daemon on OpenShift Environment Variables --------------------- -- OPENSHIFT_CELERY_BROKER_URL : The connection URL for the broker, omitting the transport. For example, an amqp value may look like: guest:guest@localhost:5672// +* OPENSHIFT_CELERY_BROKER_URL + The connection URL for the broker, omitting the transport. For example, an amqp value may look like: guest:guest@localhost:5672// -- OPENSHIFT_CELERY_IMPORTS : Defines the modules that celery should import. Currently only supports 1 module import path +* OPENSHIFT_CELERY_IMPORTS + Defines the modules that celery should import. Currently only supports 1 module import path + + +To install +--------------------- + + rhc cartridge-add https://raw.github.com/wassname/openshift-celery-cartridge/master/metadata/manifest.yml -a + +Any log output will be generated to ${OPENSHIFT_TMP_DIR}/celery_log.txt + +To manage +--------------------- + + $ rhc cartridge-status redis -a -Currently only supports MongoDB as backend for queues, but in progress to support more environment variables to allow vairous backends and configurations. -To install rhc cartridge-add https://raw.github.com/wassname/openshift-celery-cartridge/master/metadata/manifest.yml -a diff --git a/metadata/manifest.yml b/metadata/manifest.yml index 245f249..828eb35 100644 --- a/metadata/manifest.yml +++ b/metadata/manifest.yml @@ -2,13 +2,14 @@ Name: celeryd Cartridge-Short-Name: CELERY Display-Name: Celery Cartridge Description: A cartridge for running the celeryd daemon to allow asynchronous tasks in your application. -Version: '0.1' +Version: '0.15' License: ASL 2.0 Vendor: Matti HEIKKILA Cartridge-Version: - '1.0.2' -Cartridge-Vendor: customcarts -Source-Url: https://github.com/cozezien/openshift-celery-cartridge.git +Cartridge-Vendor: wassname +Website: https://github.com/wassname/openshift-celery-cartridge +Source-Url: https://github.com/wassname/openshift-celery-cartridge.git Categories: - service Requires: python-2.7 From 25c2722c840f9f9b132fdd924112e8a1c85da3ae Mon Sep 17 00:00:00 2001 From: Is Isilon Date: Fri, 10 Apr 2015 15:36:37 +0800 Subject: [PATCH 57/85] Updating --- bin/control | 7 +++---- metadata/manifest.yml | 6 ++++++ 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/bin/control b/bin/control index 2736fa3..fcd6768 100755 --- a/bin/control +++ b/bin/control @@ -5,10 +5,9 @@ PATH=${OPENSHIFT_CELERY_DIR}bin/:${OPENSHIFT_CELERY_DIR}usr/bin:$PATH #TODO add scaling, better status, variable worker names function start { - export PYTHONPATH=$OPENSHIFT_REPO_DIR/.openshift:$OPENSHIFT_REPO_DIR/wsgi:$OPENSHIFT_REPO_DIR:$PYTHONPATH - echo "cartridge ready for code" - echo "${OPENSHIFT_CELERY_DIR}usr/celery worker -A ${OPENSHIFT_CELERY_PYTHON_PKG_NAME} -b ${OPENSHIFT_CELERY_BROKER_URL} --pidfile=${OPENSHIFT_CELERY_DIR}etc/celeryd.pid --loglevel=DEBUG --logfile=${OPENSHIFT_TMP_DIR}celery_log.txt" >> `echo $OPENSHIFT_TMP_DIR`blarg.txt - ${OPENSHIFT_CELERY_DIR}usr/celery multi start worker --app=${OPENSHIFT_CELERY_IMPORTS} --broker=${OPENSHIFT_CELERY_BROKER_URL} --pidfile=${OPENSHIFT_CELERY_DIR}etc/celeryd.pid --hostname=${OPENSHIFT_APP_DNS} --loglevel=DEBUG --logfile=${OPENSHIFT_TMP_DIR}celery_log.txt + echo "Starting" + echo "${OPENSHIFT_CELERY_DIR}usr/celery worker -A ${OPENSHIFT_CELERY_IMPORTS=OPENSHIFT_APP_NAME} -b ${OPENSHIFT_CELERY_BROKER_URL} --pidfile=${OPENSHIFT_CELERY_DIR}etc/celeryd.pid --loglevel=DEBUG --logfile=${OPENSHIFT_TMP_DIR}celery_log.txt" >> `echo $OPENSHIFT_TMP_DIR`blarg.txt + ${OPENSHIFT_CELERY_DIR}usr/celery multi start worker --app=${OPENSHIFT_CELERY_IMPORTS=OPENSHIFT_APP_NAME} --broker=${OPENSHIFT_CELERY_BROKER_URL="amqp://guest@localhost//"} --pidfile=${OPENSHIFT_CELERY_DIR}etc/celeryd.pid --hostname=${OPENSHIFT_APP_DNS} --loglevel=DEBUG --logfile=${OPENSHIFT_TMP_DIR}celery_log.txt } function stop { diff --git a/metadata/manifest.yml b/metadata/manifest.yml index 828eb35..45da873 100644 --- a/metadata/manifest.yml +++ b/metadata/manifest.yml @@ -19,6 +19,12 @@ Cart-Data: - Key: OPENSHIFT_CELERY_HOME Type: environment Description: An environment variable for the Home dir of the binary + - Key: OPENSHIFT_CELERY_BROKER_URL + Type: environment + Description: An environment variable for the celery broker + - Key: OPENSHIFT_CELERY_IMPORTS + Type: environment + Description: An environment variable for the python modules for celery to import Group-Overrides: - components: - celery From 8606dcec098d01eb7c80a7463129b053660d17f9 Mon Sep 17 00:00:00 2001 From: Is Isilon Date: Fri, 10 Apr 2015 15:39:14 +0800 Subject: [PATCH 58/85] Update --- bin/control | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/control b/bin/control index fcd6768..c82c5db 100755 --- a/bin/control +++ b/bin/control @@ -7,7 +7,7 @@ PATH=${OPENSHIFT_CELERY_DIR}bin/:${OPENSHIFT_CELERY_DIR}usr/bin:$PATH function start { echo "Starting" echo "${OPENSHIFT_CELERY_DIR}usr/celery worker -A ${OPENSHIFT_CELERY_IMPORTS=OPENSHIFT_APP_NAME} -b ${OPENSHIFT_CELERY_BROKER_URL} --pidfile=${OPENSHIFT_CELERY_DIR}etc/celeryd.pid --loglevel=DEBUG --logfile=${OPENSHIFT_TMP_DIR}celery_log.txt" >> `echo $OPENSHIFT_TMP_DIR`blarg.txt - ${OPENSHIFT_CELERY_DIR}usr/celery multi start worker --app=${OPENSHIFT_CELERY_IMPORTS=OPENSHIFT_APP_NAME} --broker=${OPENSHIFT_CELERY_BROKER_URL="amqp://guest@localhost//"} --pidfile=${OPENSHIFT_CELERY_DIR}etc/celeryd.pid --hostname=${OPENSHIFT_APP_DNS} --loglevel=DEBUG --logfile=${OPENSHIFT_TMP_DIR}celery_log.txt + ${OPENSHIFT_CELERY_DIR}usr/celery multi start worker --app=${OPENSHIFT_CELERY_IMPORTS=${OPENSHIFT_APP_NAME}} --broker=${OPENSHIFT_CELERY_BROKER_URL="amqp://guest@localhost//"} --pidfile=${OPENSHIFT_CELERY_DIR}etc/celeryd.pid --hostname=${OPENSHIFT_APP_DNS} --loglevel=DEBUG --logfile=${OPENSHIFT_TMP_DIR}celery_log.txt } function stop { From e4f1b01b8e26617435b7726ddf0b7b4efb5477cd Mon Sep 17 00:00:00 2001 From: Is Isilon Date: Fri, 10 Apr 2015 15:42:27 +0800 Subject: [PATCH 59/85] Update --- bin/control | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/bin/control b/bin/control index c82c5db..1f1ca3c 100755 --- a/bin/control +++ b/bin/control @@ -5,9 +5,14 @@ PATH=${OPENSHIFT_CELERY_DIR}bin/:${OPENSHIFT_CELERY_DIR}usr/bin:$PATH #TODO add scaling, better status, variable worker names function start { - echo "Starting" - echo "${OPENSHIFT_CELERY_DIR}usr/celery worker -A ${OPENSHIFT_CELERY_IMPORTS=OPENSHIFT_APP_NAME} -b ${OPENSHIFT_CELERY_BROKER_URL} --pidfile=${OPENSHIFT_CELERY_DIR}etc/celeryd.pid --loglevel=DEBUG --logfile=${OPENSHIFT_TMP_DIR}celery_log.txt" >> `echo $OPENSHIFT_TMP_DIR`blarg.txt - ${OPENSHIFT_CELERY_DIR}usr/celery multi start worker --app=${OPENSHIFT_CELERY_IMPORTS=${OPENSHIFT_APP_NAME}} --broker=${OPENSHIFT_CELERY_BROKER_URL="amqp://guest@localhost//"} --pidfile=${OPENSHIFT_CELERY_DIR}etc/celeryd.pid --hostname=${OPENSHIFT_APP_DNS} --loglevel=DEBUG --logfile=${OPENSHIFT_TMP_DIR}celery_log.txt + if env | grep -q ^OPENSHIFT_CELERY_IMPORTS= + then + echo "Starting" + echo "${OPENSHIFT_CELERY_DIR}usr/celery worker -A ${OPENSHIFT_CELERY_IMPORTS=OPENSHIFT_APP_NAME} -b ${OPENSHIFT_CELERY_BROKER_URL} --pidfile=${OPENSHIFT_CELERY_DIR}etc/celeryd.pid --loglevel=DEBUG --logfile=${OPENSHIFT_TMP_DIR}celery_log.txt" >> `echo $OPENSHIFT_TMP_DIR`blarg.txt + ${OPENSHIFT_CELERY_DIR}usr/celery multi start worker --app=${OPENSHIFT_CELERY_IMPORTS=${OPENSHIFT_APP_NAME}} --broker=${OPENSHIFT_CELERY_BROKER_URL="amqp://guest@localhost//"} --pidfile=${OPENSHIFT_CELERY_DIR}etc/celeryd.pid --hostname=${OPENSHIFT_APP_DNS} --loglevel=DEBUG --logfile=${OPENSHIFT_TMP_DIR}celery_log.txt + else + echo 'Cant start celert as OPENSHIFT_CELERY_IMPORTS has not been set' + fi } function stop { From 24125eea57d0e57725f8f99c2d7e68258d46b2fe Mon Sep 17 00:00:00 2001 From: Is Isilon Date: Fri, 10 Apr 2015 16:11:25 +0800 Subject: [PATCH 60/85] Update --- README.md | 5 +++++ bin/control | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 661fb82..b11a26a 100644 --- a/README.md +++ b/README.md @@ -20,6 +20,11 @@ To install Any log output will be generated to ${OPENSHIFT_TMP_DIR}/celery_log.txt +This was tested using: + + celery==3.1.11 + redis==2.10.3 + To manage --------------------- diff --git a/bin/control b/bin/control index 1f1ca3c..f49ed39 100755 --- a/bin/control +++ b/bin/control @@ -11,7 +11,7 @@ function start { echo "${OPENSHIFT_CELERY_DIR}usr/celery worker -A ${OPENSHIFT_CELERY_IMPORTS=OPENSHIFT_APP_NAME} -b ${OPENSHIFT_CELERY_BROKER_URL} --pidfile=${OPENSHIFT_CELERY_DIR}etc/celeryd.pid --loglevel=DEBUG --logfile=${OPENSHIFT_TMP_DIR}celery_log.txt" >> `echo $OPENSHIFT_TMP_DIR`blarg.txt ${OPENSHIFT_CELERY_DIR}usr/celery multi start worker --app=${OPENSHIFT_CELERY_IMPORTS=${OPENSHIFT_APP_NAME}} --broker=${OPENSHIFT_CELERY_BROKER_URL="amqp://guest@localhost//"} --pidfile=${OPENSHIFT_CELERY_DIR}etc/celeryd.pid --hostname=${OPENSHIFT_APP_DNS} --loglevel=DEBUG --logfile=${OPENSHIFT_TMP_DIR}celery_log.txt else - echo 'Cant start celert as OPENSHIFT_CELERY_IMPORTS has not been set' + echo "Can't start celery as env var OPENSHIFT_CELERY_IMPORTS has not been set. More: https://developers.openshift.com/en/managing-environment-variables.html" fi } From 911798428ad52acb0bdc95f3290bbe3c1a44a596 Mon Sep 17 00:00:00 2001 From: Is Isilon Date: Fri, 10 Apr 2015 16:32:32 +0800 Subject: [PATCH 61/85] Update --- bin/control | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/bin/control b/bin/control index f49ed39..278f1de 100755 --- a/bin/control +++ b/bin/control @@ -2,14 +2,14 @@ PATH=${OPENSHIFT_CELERY_DIR}bin/:${OPENSHIFT_CELERY_DIR}usr/bin:$PATH -#TODO add scaling, better status, variable worker names +#TODO add scaling, better status, variable worker names, stop workers properly function start { if env | grep -q ^OPENSHIFT_CELERY_IMPORTS= then echo "Starting" - echo "${OPENSHIFT_CELERY_DIR}usr/celery worker -A ${OPENSHIFT_CELERY_IMPORTS=OPENSHIFT_APP_NAME} -b ${OPENSHIFT_CELERY_BROKER_URL} --pidfile=${OPENSHIFT_CELERY_DIR}etc/celeryd.pid --loglevel=DEBUG --logfile=${OPENSHIFT_TMP_DIR}celery_log.txt" >> `echo $OPENSHIFT_TMP_DIR`blarg.txt - ${OPENSHIFT_CELERY_DIR}usr/celery multi start worker --app=${OPENSHIFT_CELERY_IMPORTS=${OPENSHIFT_APP_NAME}} --broker=${OPENSHIFT_CELERY_BROKER_URL="amqp://guest@localhost//"} --pidfile=${OPENSHIFT_CELERY_DIR}etc/celeryd.pid --hostname=${OPENSHIFT_APP_DNS} --loglevel=DEBUG --logfile=${OPENSHIFT_TMP_DIR}celery_log.txt + echo "${OPENSHIFT_CELERY_DIR}usr/celery multi start worker --app=${OPENSHIFT_CELERY_IMPORTS=${OPENSHIFT_APP_NAME}} --broker=${OPENSHIFT_CELERY_BROKER_URL="amqp://guest@localhost//"} --pidfile=${OPENSHIFT_CELERY_DIR}etc/celeryd.pid --hostname=${OPENSHIFT_APP_DNS} --loglevel=DEBUG --logfile=${OPENSHIFT_TMP_DIR}celery_log.txt" >> `echo $OPENSHIFT_TMP_DIR`celery_log.txt + ${OPENSHIFT_CELERY_DIR}usr/celery multi start worker --app=${OPENSHIFT_CELERY_IMPORTS} --broker=${OPENSHIFT_CELERY_BROKER_URL="amqp://guest@localhost//"} --pidfile=${OPENSHIFT_CELERY_DIR}etc/celeryd.pid --hostname=${OPENSHIFT_APP_DNS} --loglevel=DEBUG --logfile=${OPENSHIFT_TMP_DIR}celery_log.txt else echo "Can't start celery as env var OPENSHIFT_CELERY_IMPORTS has not been set. More: https://developers.openshift.com/en/managing-environment-variables.html" fi From 0d25c4d35270f4717b76e8008394e845b7d6d4fe Mon Sep 17 00:00:00 2001 From: Is Isilon Date: Fri, 10 Apr 2015 18:45:11 +0800 Subject: [PATCH 62/85] Update --- README.md | 10 +++------- bin/control | 13 ++++--------- conf.d/celeryconfig.py | 20 ++++++++++++++++++++ 3 files changed, 27 insertions(+), 16 deletions(-) create mode 100644 conf.d/celeryconfig.py diff --git a/README.md b/README.md index b11a26a..ba1b216 100644 --- a/README.md +++ b/README.md @@ -3,15 +3,11 @@ openshift-celery-cartridge Cartridge to Expose Celery as a Daemon on OpenShift -Environment Variables +Configuration --------------------- -* OPENSHIFT_CELERY_BROKER_URL - The connection URL for the broker, omitting the transport. For example, an amqp value may look like: guest:guest@localhost:5672// - -* OPENSHIFT_CELERY_IMPORTS - Defines the modules that celery should import. Currently only supports 1 module import path - +* OPENSHIFT_CELERY_CONFIG + This is the name of your config file, which by default is $OPENSHIFT_CELERY_DIR/conf.d/celeryconfig.py To install --------------------- diff --git a/bin/control b/bin/control index 278f1de..2797382 100755 --- a/bin/control +++ b/bin/control @@ -1,18 +1,13 @@ #!/bin/bash -e -PATH=${OPENSHIFT_CELERY_DIR}bin/:${OPENSHIFT_CELERY_DIR}usr/bin:$PATH +PATH=${OPENSHIFT_CELERY_DIR}bin/:${OPENSHIFT_CELERY_DIR}usr/bin:${OPENSHIFT_CELERY_DIR}conf.d:$PATH #TODO add scaling, better status, variable worker names, stop workers properly function start { - if env | grep -q ^OPENSHIFT_CELERY_IMPORTS= - then - echo "Starting" - echo "${OPENSHIFT_CELERY_DIR}usr/celery multi start worker --app=${OPENSHIFT_CELERY_IMPORTS=${OPENSHIFT_APP_NAME}} --broker=${OPENSHIFT_CELERY_BROKER_URL="amqp://guest@localhost//"} --pidfile=${OPENSHIFT_CELERY_DIR}etc/celeryd.pid --hostname=${OPENSHIFT_APP_DNS} --loglevel=DEBUG --logfile=${OPENSHIFT_TMP_DIR}celery_log.txt" >> `echo $OPENSHIFT_TMP_DIR`celery_log.txt - ${OPENSHIFT_CELERY_DIR}usr/celery multi start worker --app=${OPENSHIFT_CELERY_IMPORTS} --broker=${OPENSHIFT_CELERY_BROKER_URL="amqp://guest@localhost//"} --pidfile=${OPENSHIFT_CELERY_DIR}etc/celeryd.pid --hostname=${OPENSHIFT_APP_DNS} --loglevel=DEBUG --logfile=${OPENSHIFT_TMP_DIR}celery_log.txt - else - echo "Can't start celery as env var OPENSHIFT_CELERY_IMPORTS has not been set. More: https://developers.openshift.com/en/managing-environment-variables.html" - fi + echo "Starting celery:${OPENSHIFT_CELERY_DIR}usr/celery multi start worker --configfile=celeryconfig" + #echo "${OPENSHIFT_CELERY_DIR}usr/celery multi start worker --app=${OPENSHIFT_CELERY_IMPORTS=${OPENSHIFT_APP_NAME}} --broker=${OPENSHIFT_CELERY_BROKER_URL="amqp://guest@localhost//"} --pidfile=${OPENSHIFT_CELERY_DIR}etc/celeryd.pid --hostname=${OPENSHIFT_APP_DNS} --loglevel=DEBUG --logfile=${OPENSHIFT_TMP_DIR}celery_log.txt" >> `echo $OPENSHIFT_TMP_DIR`celery_log.txt + ${OPENSHIFT_CELERY_DIR}usr/celery multi start worker --configfile=$(OPENSHIFT_CELERY_CONFIG=celeryconfig} } function stop { diff --git a/conf.d/celeryconfig.py b/conf.d/celeryconfig.py new file mode 100644 index 0000000..daae2e1 --- /dev/null +++ b/conf.d/celeryconfig.py @@ -0,0 +1,20 @@ +# Modify this file to configure celery. +# You can move it to your data directory and reference it in enviromental variable $OPENSHIFT_CELERY_CONFIG. +# You could also insert variables at build using an acton hook. +# config options here http://docs.celeryproject.org/en/2.5/configuration.html#worker-celeryd + +import os +import sys + +sys.path.append('.') + +BROKER_HOST = "localhost" +BROKER_PORT = 5672 +BROKER_USER = "celeryuser" +BROKER_PASSWORD = "celery" +BROKER_VHOST = "celeryvhost" + +CELERY_RESULT_BACKEND = "amqp" + +# Here we check the env vars first, otherwise go with "tasks" +CELERY_IMPORTS = ( os.getenv("OPENSHIFT_CELERY_IMPORTS","tasks") ,) From ca5624cf9180ee0567be493f204ca8e0d971892f Mon Sep 17 00:00:00 2001 From: Is Isilon Date: Fri, 10 Apr 2015 18:49:54 +0800 Subject: [PATCH 63/85] Update --- bin/control | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/bin/control b/bin/control index 2797382..4dce946 100755 --- a/bin/control +++ b/bin/control @@ -5,9 +5,8 @@ PATH=${OPENSHIFT_CELERY_DIR}bin/:${OPENSHIFT_CELERY_DIR}usr/bin:${OPENSHIFT_CELE #TODO add scaling, better status, variable worker names, stop workers properly function start { - echo "Starting celery:${OPENSHIFT_CELERY_DIR}usr/celery multi start worker --configfile=celeryconfig" - #echo "${OPENSHIFT_CELERY_DIR}usr/celery multi start worker --app=${OPENSHIFT_CELERY_IMPORTS=${OPENSHIFT_APP_NAME}} --broker=${OPENSHIFT_CELERY_BROKER_URL="amqp://guest@localhost//"} --pidfile=${OPENSHIFT_CELERY_DIR}etc/celeryd.pid --hostname=${OPENSHIFT_APP_DNS} --loglevel=DEBUG --logfile=${OPENSHIFT_TMP_DIR}celery_log.txt" >> `echo $OPENSHIFT_TMP_DIR`celery_log.txt - ${OPENSHIFT_CELERY_DIR}usr/celery multi start worker --configfile=$(OPENSHIFT_CELERY_CONFIG=celeryconfig} + echo "Starting ${OPENSHIFT_CELERY_DIR}usr/celery multi start worker --configfile=${OPENSHIFT_CELERY_CONFIG=celeryconfig} --loglevel=DEBUG --logfile=${OPENSHIFT_TMP_DIR}celery_log.txt" + ${OPENSHIFT_CELERY_DIR}usr/celery multi start worker --configfile=${OPENSHIFT_CELERY_CONFIG=celeryconfig} --loglevel=DEBUG --logfile=${OPENSHIFT_TMP_DIR}celery_log.txt } function stop { From d914107df8ec8120295438cd3e306660a7b7f88f Mon Sep 17 00:00:00 2001 From: Is Isilon Date: Fri, 10 Apr 2015 19:23:50 +0800 Subject: [PATCH 64/85] Update --- bin/control | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/control b/bin/control index 4dce946..3c5ae31 100755 --- a/bin/control +++ b/bin/control @@ -1,6 +1,6 @@ #!/bin/bash -e -PATH=${OPENSHIFT_CELERY_DIR}bin/:${OPENSHIFT_CELERY_DIR}usr/bin:${OPENSHIFT_CELERY_DIR}conf.d:$PATH +PATH=${OPENSHIFT_CELERY_DIR}bin/:${OPENSHIFT_CELERY_DIR}usr/bin:${OPENSHIFT_CELERY_DIR}conf.d:$OPENSHIFT_DATA_DIR:$OPENSHIFT_DATA_DIR/config/:$PATH: #TODO add scaling, better status, variable worker names, stop workers properly From e6c9f78616eb4a5b40dc1da3d780ff32ebf326f2 Mon Sep 17 00:00:00 2001 From: Is Isilon Date: Fri, 10 Apr 2015 19:35:48 +0800 Subject: [PATCH 65/85] Update --- bin/control | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bin/control b/bin/control index 3c5ae31..acf2855 100755 --- a/bin/control +++ b/bin/control @@ -1,11 +1,11 @@ #!/bin/bash -e -PATH=${OPENSHIFT_CELERY_DIR}bin/:${OPENSHIFT_CELERY_DIR}usr/bin:${OPENSHIFT_CELERY_DIR}conf.d:$OPENSHIFT_DATA_DIR:$OPENSHIFT_DATA_DIR/config/:$PATH: +PATH=${OPENSHIFT_CELERY_DIR}bin/:${OPENSHIFT_CELERY_DIR}usr/bin:${OPENSHIFT_CELERY_DIR}conf.d:$OPENSHIFT_DATA_DIR:$OPENSHIFT_DATA_DIR/config/:$PATH #TODO add scaling, better status, variable worker names, stop workers properly function start { - echo "Starting ${OPENSHIFT_CELERY_DIR}usr/celery multi start worker --configfile=${OPENSHIFT_CELERY_CONFIG=celeryconfig} --loglevel=DEBUG --logfile=${OPENSHIFT_TMP_DIR}celery_log.txt" + echo "Starting ${OPENSHIFT_CELERY_DIR}usr/celery multi start worker --config=${OPENSHIFT_CELERY_CONFIG=celeryconfig} --loglevel=DEBUG --logfile=${OPENSHIFT_TMP_DIR}celery_log.txt" ${OPENSHIFT_CELERY_DIR}usr/celery multi start worker --configfile=${OPENSHIFT_CELERY_CONFIG=celeryconfig} --loglevel=DEBUG --logfile=${OPENSHIFT_TMP_DIR}celery_log.txt } From 5e96f86fa20c437ef09d902e8064588f637a29ee Mon Sep 17 00:00:00 2001 From: Is Isilon Date: Fri, 10 Apr 2015 21:39:04 +0800 Subject: [PATCH 66/85] Update --- bin/control | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/control b/bin/control index acf2855..3f919e8 100755 --- a/bin/control +++ b/bin/control @@ -6,7 +6,7 @@ PATH=${OPENSHIFT_CELERY_DIR}bin/:${OPENSHIFT_CELERY_DIR}usr/bin:${OPENSHIFT_CELE function start { echo "Starting ${OPENSHIFT_CELERY_DIR}usr/celery multi start worker --config=${OPENSHIFT_CELERY_CONFIG=celeryconfig} --loglevel=DEBUG --logfile=${OPENSHIFT_TMP_DIR}celery_log.txt" - ${OPENSHIFT_CELERY_DIR}usr/celery multi start worker --configfile=${OPENSHIFT_CELERY_CONFIG=celeryconfig} --loglevel=DEBUG --logfile=${OPENSHIFT_TMP_DIR}celery_log.txt + ${OPENSHIFT_CELERY_DIR}usr/celery multi start worker --config=${OPENSHIFT_CELERY_CONFIG=celeryconfig} --loglevel=DEBUG --logfile=${OPENSHIFT_TMP_DIR}celery_log.txt } function stop { From 5a7f69299024ee3abde19f76e169b5141ce4ad48 Mon Sep 17 00:00:00 2001 From: Is Isilon Date: Fri, 10 Apr 2015 23:49:58 +0800 Subject: [PATCH 67/85] Update --- README.md | 2 +- bin/control | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index ba1b216..2751286 100644 --- a/README.md +++ b/README.md @@ -14,7 +14,7 @@ To install rhc cartridge-add https://raw.github.com/wassname/openshift-celery-cartridge/master/metadata/manifest.yml -a -Any log output will be generated to ${OPENSHIFT_TMP_DIR}/celery_log.txt +Any log output will be generated to ${OPENSHIFT_APP_ROOT}logs/celery_log.txt This was tested using: diff --git a/bin/control b/bin/control index 3f919e8..7afaac9 100755 --- a/bin/control +++ b/bin/control @@ -6,7 +6,7 @@ PATH=${OPENSHIFT_CELERY_DIR}bin/:${OPENSHIFT_CELERY_DIR}usr/bin:${OPENSHIFT_CELE function start { echo "Starting ${OPENSHIFT_CELERY_DIR}usr/celery multi start worker --config=${OPENSHIFT_CELERY_CONFIG=celeryconfig} --loglevel=DEBUG --logfile=${OPENSHIFT_TMP_DIR}celery_log.txt" - ${OPENSHIFT_CELERY_DIR}usr/celery multi start worker --config=${OPENSHIFT_CELERY_CONFIG=celeryconfig} --loglevel=DEBUG --logfile=${OPENSHIFT_TMP_DIR}celery_log.txt + ${OPENSHIFT_CELERY_DIR}usr/celery multi start worker --config=${OPENSHIFT_CELERY_CONFIG=celeryconfig} --loglevel=DEBUG --logfile=${OPENSHIFT_APP_ROOT}app-root/logs/celery_log.txt } function stop { From ca09418f28f740b2c1bfdab4651eda38b19cdaab Mon Sep 17 00:00:00 2001 From: Is Isilon Date: Sat, 11 Apr 2015 00:13:21 +0800 Subject: [PATCH 68/85] Update --- bin/control | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/control b/bin/control index 7afaac9..339a6c7 100755 --- a/bin/control +++ b/bin/control @@ -6,7 +6,7 @@ PATH=${OPENSHIFT_CELERY_DIR}bin/:${OPENSHIFT_CELERY_DIR}usr/bin:${OPENSHIFT_CELE function start { echo "Starting ${OPENSHIFT_CELERY_DIR}usr/celery multi start worker --config=${OPENSHIFT_CELERY_CONFIG=celeryconfig} --loglevel=DEBUG --logfile=${OPENSHIFT_TMP_DIR}celery_log.txt" - ${OPENSHIFT_CELERY_DIR}usr/celery multi start worker --config=${OPENSHIFT_CELERY_CONFIG=celeryconfig} --loglevel=DEBUG --logfile=${OPENSHIFT_APP_ROOT}app-root/logs/celery_log.txt + ${OPENSHIFT_CELERY_DIR}usr/celery multi start worker --config=${OPENSHIFT_CELERY_CONFIG=celeryconfig} --loglevel=DEBUG --pidfile=${OPENSHIFT_CELERY_DIR}etc/celeryd.pid --logfile=${OPENSHIFT_APP_ROOT}app-root/logs/celery_log.txt } function stop { From 5952af31234148f3a7e88cd97bffc2653a338ef7 Mon Sep 17 00:00:00 2001 From: Is Isilon Date: Sat, 11 Apr 2015 00:22:35 +0800 Subject: [PATCH 69/85] Update --- bin/control | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/bin/control b/bin/control index 339a6c7..f8fa308 100755 --- a/bin/control +++ b/bin/control @@ -1,12 +1,14 @@ #!/bin/bash -e -PATH=${OPENSHIFT_CELERY_DIR}bin/:${OPENSHIFT_CELERY_DIR}usr/bin:${OPENSHIFT_CELERY_DIR}conf.d:$OPENSHIFT_DATA_DIR:$OPENSHIFT_DATA_DIR/config/:$PATH +PATH=$OPENSHIFT_DATA_DIR:$OPENSHIFT_DATA_DIR/config/:${OPENSHIFT_CELERY_DIR}bin/:${OPENSHIFT_CELERY_DIR}usr/bin:${OPENSHIFT_CELERY_DIR}conf.d:$PATH #TODO add scaling, better status, variable worker names, stop workers properly function start { + export PYTHONPATH=$OPENSHIFT_DATA_DIR:$OPENSHIFT_DATA_DIR/config/:$OPENSHIFT_REPO_DIR/.openshift:$OPENSHIFT_REPO_DIR/wsgi::$OPENSHIFT_CELERY_DIR/conf.d:$OPENSHIFT_REPO_DIR:$PYTHONPATH + PYTHONPATH=$OPENSHIFT_DATA_DIR:$OPENSHIFT_DATA_DIR/config/:$OPENSHIFT_REPO_DIR/.openshift:$OPENSHIFT_REPO_DIR/wsgi::$OPENSHIFT_CELERY_DIR/conf.d:$OPENSHIFT_REPO_DIR:$PYTHONPATH echo "Starting ${OPENSHIFT_CELERY_DIR}usr/celery multi start worker --config=${OPENSHIFT_CELERY_CONFIG=celeryconfig} --loglevel=DEBUG --logfile=${OPENSHIFT_TMP_DIR}celery_log.txt" - ${OPENSHIFT_CELERY_DIR}usr/celery multi start worker --config=${OPENSHIFT_CELERY_CONFIG=celeryconfig} --loglevel=DEBUG --pidfile=${OPENSHIFT_CELERY_DIR}etc/celeryd.pid --logfile=${OPENSHIFT_APP_ROOT}app-root/logs/celery_log.txt + ${OPENSHIFT_CELERY_DIR}usr/celery multi start worker --config=${OPENSHIFT_CELERY_CONFIG=celeryconfig} --loglevel=DEBUG --hostname=$OPENSHIFT_APP_DNS --pidfile=${OPENSHIFT_CELERY_DIR}etc/celeryd.pid --logfile=${OPENSHIFT_APP_ROOT}app-root/logs/celery_log.txt } function stop { From c7320d0a965fa082d0c9139df0240d7722e1956b Mon Sep 17 00:00:00 2001 From: Is Isilon Date: Sat, 11 Apr 2015 00:38:29 +0800 Subject: [PATCH 70/85] Update --- README.md | 6 +++--- bin/control | 3 ++- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 2751286..1ac21ca 100644 --- a/README.md +++ b/README.md @@ -7,14 +7,14 @@ Configuration --------------------- * OPENSHIFT_CELERY_CONFIG - This is the name of your config file, which by default is $OPENSHIFT_CELERY_DIR/conf.d/celeryconfig.py + This is the name of your config file, which by default is $OPENSHIFT_CELERY_DIR/conf.d/celeryconfig.py but you can copy this to $OPENSHIFT_DATA_DIR/config/celeryconfig.py and edit your own copy. To install --------------------- rhc cartridge-add https://raw.github.com/wassname/openshift-celery-cartridge/master/metadata/manifest.yml -a -Any log output will be generated to ${OPENSHIFT_APP_ROOT}logs/celery_log.txt +Any log output will be generated to ${OPENSHIFT_APP_ROOT}logs/celery_log.txt and will be viewable with the rhc tail command This was tested using: @@ -24,6 +24,6 @@ This was tested using: To manage --------------------- - $ rhc cartridge-status redis -a + $ rhc cartridge-status celeryd -a diff --git a/bin/control b/bin/control index f8fa308..3f7b5c9 100755 --- a/bin/control +++ b/bin/control @@ -7,7 +7,8 @@ PATH=$OPENSHIFT_DATA_DIR:$OPENSHIFT_DATA_DIR/config/:${OPENSHIFT_CELERY_DIR}bin/ function start { export PYTHONPATH=$OPENSHIFT_DATA_DIR:$OPENSHIFT_DATA_DIR/config/:$OPENSHIFT_REPO_DIR/.openshift:$OPENSHIFT_REPO_DIR/wsgi::$OPENSHIFT_CELERY_DIR/conf.d:$OPENSHIFT_REPO_DIR:$PYTHONPATH PYTHONPATH=$OPENSHIFT_DATA_DIR:$OPENSHIFT_DATA_DIR/config/:$OPENSHIFT_REPO_DIR/.openshift:$OPENSHIFT_REPO_DIR/wsgi::$OPENSHIFT_CELERY_DIR/conf.d:$OPENSHIFT_REPO_DIR:$PYTHONPATH - echo "Starting ${OPENSHIFT_CELERY_DIR}usr/celery multi start worker --config=${OPENSHIFT_CELERY_CONFIG=celeryconfig} --loglevel=DEBUG --logfile=${OPENSHIFT_TMP_DIR}celery_log.txt" + echo "Starting Celery, cmd: ${OPENSHIFT_CELERY_DIR}usr/celery multi start worker --config=${OPENSHIFT_CELERY_CONFIG=celeryconfig} --loglevel=DEBUG --hostname=$OPENSHIFT_APP_DNS --pidfile=${OPENSHIFT_CELERY_DIR}etc/celeryd.pid --logfile=${OPENSHIFT_APP_ROOT}app-root/logs/celery_log.txt" + echo `${OPENSHIFT_CELERY_DIR}usr/celery multi start worker --config=${OPENSHIFT_CELERY_CONFIG=celeryconfig} --loglevel=DEBUG --hostname=$OPENSHIFT_APP_DNS --pidfile=${OPENSHIFT_CELERY_DIR}etc/celeryd.pid --logfile=${OPENSHIFT_APP_ROOT}app-root/logs/celery_log.txt` > ${OPENSHIFT_APP_ROOT}app-root/logs/celery_log.txt ${OPENSHIFT_CELERY_DIR}usr/celery multi start worker --config=${OPENSHIFT_CELERY_CONFIG=celeryconfig} --loglevel=DEBUG --hostname=$OPENSHIFT_APP_DNS --pidfile=${OPENSHIFT_CELERY_DIR}etc/celeryd.pid --logfile=${OPENSHIFT_APP_ROOT}app-root/logs/celery_log.txt } From ce587bf222e01dc5660464c83f8a69c83d499599 Mon Sep 17 00:00:00 2001 From: Is Isilon Date: Sat, 11 Apr 2015 00:39:53 +0800 Subject: [PATCH 71/85] Update --- README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 1ac21ca..8e24d3a 100644 --- a/README.md +++ b/README.md @@ -12,9 +12,9 @@ Configuration To install --------------------- - rhc cartridge-add https://raw.github.com/wassname/openshift-celery-cartridge/master/metadata/manifest.yml -a + rhc cartridge-add https://raw.github.com/wassname/openshift-celery-cartridge/master/metadata/manifest.yml -a "appname" -Any log output will be generated to ${OPENSHIFT_APP_ROOT}logs/celery_log.txt and will be viewable with the rhc tail command +Any log output will be generated to ${OPENSHIFT_APP_ROOT}logs/celery_log.txt and will be viewable with the rhc tail "appname" command This was tested using: @@ -24,6 +24,6 @@ This was tested using: To manage --------------------- - $ rhc cartridge-status celeryd -a + $ rhc cartridge-status celeryd -a "yourapp" From 09b2b59968e59124f2feffd91538ebed4d7b2c58 Mon Sep 17 00:00:00 2001 From: Is Isilon Date: Sat, 11 Apr 2015 11:59:41 +0800 Subject: [PATCH 72/85] Scaling --- .gitignore | 2 + bin/control | 12 +++ bin/setup | 9 ++ hooks/publish-celery-connection-info | 5 ++ hooks/publish-db-connection-info | 4 + hooks/set-celery-connection-info | 87 ++++++++++++++++++++ metadata/managed_files.yml | 2 - metadata/manifest.yml | 22 ++++- template/.openshift/action_hooks/build | 1 - template/.openshift/action_hooks/deploy | 6 -- template/.openshift/action_hooks/post-deploy | 1 - template/.openshift/action_hooks/post_deploy | 4 - template/.openshift/action_hooks/pre-build | 1 - template/celerytks/__init__.py | 0 template/celerytks/tasks.py | 15 ---- 15 files changed, 139 insertions(+), 32 deletions(-) create mode 100644 .gitignore create mode 100644 bin/setup create mode 100755 hooks/publish-celery-connection-info create mode 100755 hooks/publish-db-connection-info create mode 100755 hooks/set-celery-connection-info delete mode 100644 template/.openshift/action_hooks/build delete mode 100755 template/.openshift/action_hooks/deploy delete mode 100644 template/.openshift/action_hooks/post-deploy delete mode 100755 template/.openshift/action_hooks/post_deploy delete mode 100644 template/.openshift/action_hooks/pre-build delete mode 100644 template/celerytks/__init__.py delete mode 100644 template/celerytks/tasks.py diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..ea5cf20 --- /dev/null +++ b/.gitignore @@ -0,0 +1,2 @@ +env +logs diff --git a/bin/control b/bin/control index 3f7b5c9..ee7d413 100755 --- a/bin/control +++ b/bin/control @@ -1,5 +1,7 @@ #!/bin/bash -e +source $OPENSHIFT_CARTRIDGE_SDK_BASH + PATH=$OPENSHIFT_DATA_DIR:$OPENSHIFT_DATA_DIR/config/:${OPENSHIFT_CELERY_DIR}bin/:${OPENSHIFT_CELERY_DIR}usr/bin:${OPENSHIFT_CELERY_DIR}conf.d:$PATH #TODO add scaling, better status, variable worker names, stop workers properly @@ -39,10 +41,20 @@ function status() { fi } + function catchall { echo "not yet implemented" } +# Ensure arguments. +if ! [ $# -gt 0 ]; then + echo "Usage: $0 [start|restart|stop|status]" + exit 1 +fi + +# Source utility functions. +source $OPENSHIFT_CARTRIDGE_SDK_BASH + case "$1" in start) start ;; stop) stop ;; diff --git a/bin/setup b/bin/setup new file mode 100644 index 0000000..66df359 --- /dev/null +++ b/bin/setup @@ -0,0 +1,9 @@ +#!/bin/bash -e + +source $OPENSHIFT_CARTRIDGE_SDK_BASH + +for dir in logs pid tmp env; do + mkdir -p $dir +done + +mkdir -p $OPENSHIFT_DATA_DIR/.celery diff --git a/hooks/publish-celery-connection-info b/hooks/publish-celery-connection-info new file mode 100755 index 0000000..28ce2bb --- /dev/null +++ b/hooks/publish-celery-connection-info @@ -0,0 +1,5 @@ +#!/bin/bash + +echo "S_CELERY_MASTER=$CELERY_MASTER" +echo "S_CELERY_HOST=$OPENSHIFT_GEAR_DNS" +echo "S_CELERY_PORT=$OPENSHIFT_GEAR_PORT" diff --git a/hooks/publish-db-connection-info b/hooks/publish-db-connection-info new file mode 100755 index 0000000..dc458e8 --- /dev/null +++ b/hooks/publish-db-connection-info @@ -0,0 +1,4 @@ +#!/bin/bash + +echo "OPENSHIFT_CELERY_HOST=$OPENSHIFT_GEAR_DNS" +echo "OPENSHIFT_CELERY_PORT=$OPENSHIFT_GEAR_PORT" diff --git a/hooks/set-celery-connection-info b/hooks/set-celery-connection-info new file mode 100755 index 0000000..4498507 --- /dev/null +++ b/hooks/set-celery-connection-info @@ -0,0 +1,87 @@ +#!/usr/bin/ruby + +def gear_info(tokens, gear_id, &block) + gears = {} + if tokens.length == 1 + gears[gear_id] = tokens.first + else + while not tokens.empty? + gear, delim, data = tokens.shift(3) + data = yield data if block_given? + raise "Invalid data" unless delim == '=' + gears[gear] = data + tokens.shift if tokens.first == ' ' + end + end + gears +end + +def tokenize(s) + tokens = [] + a = "" + state = :start + s.scan(/([ \t]+)|(\\')|(')|([^ \t']+)/) do |args| + space, escaped_delim, delim, text = args + case state + when :start + case + when space then " " + when escaped_delim then raise "Unexpected delimiter" + when delim then state = :within_delim + when text then tokens << text + else raise "error" + end + when :within_delim + case + when space then a << space + when escaped_delim then a << '"' + when delim then tokens << a; a = ""; state = :start + when text then a << text + end + end + end + tokens << a if a.length > 0 + tokens +end + +gear_id = ARGV.shift +domain = ARGV.shift +tokens = tokenize(ARGV.shift) + +# better if this is in creation order (oldest first) +gears = gear_info(tokens, gear_id) do |d| + d.split(' ').map{ |s| s.scan(/\A(.+?)=(.*?);?\Z/).first }.inject({}){ |h, (k,v)| h[k] = v if v != ''; h } +end +gears.each_pair{ |k,v| puts "Found gear #{k}#{k == gear_id ? '* ' : ''} with data #{v.inspect}" } +gear_ids = gears.keys.sort.uniq + +puts "-------" + +was_master = ENV['CELERY_MASTER'] == '1' +masters = gears.map{ |k,v| v['S_CELERY_MASTER'] == '1' ? k : nil }.compact.uniq.sort + +mode = + case ENV['CELERY_MODE'] + when 'read_replica' + if masters.length > 1 + masters = masters[0,1] + elsif masters.length == 0 + masters = gears.keys.uniq.sort[0,1] + end + puts "Running in read replica mode with master #{masters}" + :read_replica + else + masters = gears.keys.uniq.sort + puts "Running sharded with masters #{masters.inspect}" + :sharded + end + +# An array of all of the host:port pairs for the cluster +hosts = gears.map{ |k,v| "#{v['S_CELERY_HOST']}:#{v['S_CELERY_PORT']}" }.compact.uniq.sort +# A list of key value pairs in => for the cluster +members = gears.map{ |k,v| "#{k}=#{v['S_CELERY_HOST']}:#{v['S_CELERY_PORT']}" }.compact.uniq.sort + +File.open('env/CELERY_CLUSTER_MEMBERS', 'w'){ |f| f.puts members.join("\n") } +File.open('env/CELERY_CLUSTER', 'w'){ |f| f.puts hosts.join(",") } +File.open('env/CELERY_CLUSTER_MASTERS', 'w'){ |f| f.puts masters.join(",") } +File.open('env/CELERY_MASTER', 'w'){ |f| f.puts masters.include?(gear_id) ? "1" : "0" } diff --git a/metadata/managed_files.yml b/metadata/managed_files.yml index 24d0b94..be02f60 100644 --- a/metadata/managed_files.yml +++ b/metadata/managed_files.yml @@ -3,5 +3,3 @@ locked_files: - env/ - env/* - conf.d/* -processed_templates: -- '**/*.erb' diff --git a/metadata/manifest.yml b/metadata/manifest.yml index 45da873..633bec2 100644 --- a/metadata/manifest.yml +++ b/metadata/manifest.yml @@ -2,7 +2,7 @@ Name: celeryd Cartridge-Short-Name: CELERY Display-Name: Celery Cartridge Description: A cartridge for running the celeryd daemon to allow asynchronous tasks in your application. -Version: '0.15' +Version: '0.16' License: ASL 2.0 Vendor: Matti HEIKKILA Cartridge-Version: @@ -12,9 +12,18 @@ Website: https://github.com/wassname/openshift-celery-cartridge Source-Url: https://github.com/wassname/openshift-celery-cartridge.git Categories: - service -Requires: python-2.7 + - embedded +Requires: celery +Scaling: + Min: 1 + Max: -1 Provides: - celery +Endpoints: + - Private-IP-Name: HOST + Private-Port-Name: PORT + Private-Port: 16388 + Public-Port-Name: PROXY_PORT Cart-Data: - Key: OPENSHIFT_CELERY_HOME Type: environment @@ -28,3 +37,12 @@ Cart-Data: Group-Overrides: - components: - celery +Subscribes: + set-redis-connection-info: + Type: "ENV:NET_TCP:db:celery:connection-info-v1" + Required: false +Publishes: + publish-db-connection-info: + Type: "ENV:NET_TCP:db:connection-info" + publish-redis-connection-info: + Type: "ENV:NET_TCP:db:celery:connection-info-v1" diff --git a/template/.openshift/action_hooks/build b/template/.openshift/action_hooks/build deleted file mode 100644 index 8b13789..0000000 --- a/template/.openshift/action_hooks/build +++ /dev/null @@ -1 +0,0 @@ - diff --git a/template/.openshift/action_hooks/deploy b/template/.openshift/action_hooks/deploy deleted file mode 100755 index 3875983..0000000 --- a/template/.openshift/action_hooks/deploy +++ /dev/null @@ -1,6 +0,0 @@ -#!/bin/bash -# This deploy hook gets executed after dependencies are resolved and the -# build hook has been run but before the application has been started back -# up again. This script gets executed directly, so it could be python, php, -# ruby, etc. - diff --git a/template/.openshift/action_hooks/post-deploy b/template/.openshift/action_hooks/post-deploy deleted file mode 100644 index 8b13789..0000000 --- a/template/.openshift/action_hooks/post-deploy +++ /dev/null @@ -1 +0,0 @@ - diff --git a/template/.openshift/action_hooks/post_deploy b/template/.openshift/action_hooks/post_deploy deleted file mode 100755 index d38c5da..0000000 --- a/template/.openshift/action_hooks/post_deploy +++ /dev/null @@ -1,4 +0,0 @@ -#!/bin/bash - -source $OPENSHIFT_HOMEDIR/python/virtenv/bin/activate -export PYTHONPATH=`echo $OPENSHIFT_REPO_DIR`:$PYTHONPATH diff --git a/template/.openshift/action_hooks/pre-build b/template/.openshift/action_hooks/pre-build deleted file mode 100644 index 8b13789..0000000 --- a/template/.openshift/action_hooks/pre-build +++ /dev/null @@ -1 +0,0 @@ - diff --git a/template/celerytks/__init__.py b/template/celerytks/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/template/celerytks/tasks.py b/template/celerytks/tasks.py deleted file mode 100644 index ac79360..0000000 --- a/template/celerytks/tasks.py +++ /dev/null @@ -1,15 +0,0 @@ -import celery - -app = celery.Celery() -app.config_from_object('celeryconfig') - -@app.task -def addtest(x, y): - return x + y - -@app.task -def run_spiders(name, **kwargs): - """ - @param name: spider name - """ - pass From 0401248d73d24f4c19ec64e2d2795e68a56e5238 Mon Sep 17 00:00:00 2001 From: Is Isilon Date: Sat, 11 Apr 2015 12:03:25 +0800 Subject: [PATCH 73/85] Scaling --- bin/setup | 0 1 file changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 bin/setup diff --git a/bin/setup b/bin/setup old mode 100644 new mode 100755 From 879a45ebd3892ac9665433ebb094c55da6bf69ac Mon Sep 17 00:00:00 2001 From: Is Isilon Date: Sat, 11 Apr 2015 13:07:18 +0800 Subject: [PATCH 74/85] finishing --- README.md | 8 ++++++-- bin/control | 6 +++--- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 8e24d3a..9cd565f 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,11 @@ openshift-celery-cartridge ========================== -Cartridge to Expose Celery as a Daemon on OpenShift +Cartridge to Expose Celery as a Daemon on OpenShift. + +Status +--------------------- +This work on unscaled applications but doesn't work as a gearin a scaled application. Further development can combine this with a python cartridge so it can operate in it's own gears. Configuration --------------------- @@ -14,7 +18,7 @@ To install rhc cartridge-add https://raw.github.com/wassname/openshift-celery-cartridge/master/metadata/manifest.yml -a "appname" -Any log output will be generated to ${OPENSHIFT_APP_ROOT}logs/celery_log.txt and will be viewable with the rhc tail "appname" command +Any log output will be generated to ${OPENSHIFT_HOMEDIR}logs/celery_log.txt and will be viewable with the rhc tail "appname" command This was tested using: diff --git a/bin/control b/bin/control index ee7d413..b368495 100755 --- a/bin/control +++ b/bin/control @@ -9,9 +9,9 @@ PATH=$OPENSHIFT_DATA_DIR:$OPENSHIFT_DATA_DIR/config/:${OPENSHIFT_CELERY_DIR}bin/ function start { export PYTHONPATH=$OPENSHIFT_DATA_DIR:$OPENSHIFT_DATA_DIR/config/:$OPENSHIFT_REPO_DIR/.openshift:$OPENSHIFT_REPO_DIR/wsgi::$OPENSHIFT_CELERY_DIR/conf.d:$OPENSHIFT_REPO_DIR:$PYTHONPATH PYTHONPATH=$OPENSHIFT_DATA_DIR:$OPENSHIFT_DATA_DIR/config/:$OPENSHIFT_REPO_DIR/.openshift:$OPENSHIFT_REPO_DIR/wsgi::$OPENSHIFT_CELERY_DIR/conf.d:$OPENSHIFT_REPO_DIR:$PYTHONPATH - echo "Starting Celery, cmd: ${OPENSHIFT_CELERY_DIR}usr/celery multi start worker --config=${OPENSHIFT_CELERY_CONFIG=celeryconfig} --loglevel=DEBUG --hostname=$OPENSHIFT_APP_DNS --pidfile=${OPENSHIFT_CELERY_DIR}etc/celeryd.pid --logfile=${OPENSHIFT_APP_ROOT}app-root/logs/celery_log.txt" - echo `${OPENSHIFT_CELERY_DIR}usr/celery multi start worker --config=${OPENSHIFT_CELERY_CONFIG=celeryconfig} --loglevel=DEBUG --hostname=$OPENSHIFT_APP_DNS --pidfile=${OPENSHIFT_CELERY_DIR}etc/celeryd.pid --logfile=${OPENSHIFT_APP_ROOT}app-root/logs/celery_log.txt` > ${OPENSHIFT_APP_ROOT}app-root/logs/celery_log.txt - ${OPENSHIFT_CELERY_DIR}usr/celery multi start worker --config=${OPENSHIFT_CELERY_CONFIG=celeryconfig} --loglevel=DEBUG --hostname=$OPENSHIFT_APP_DNS --pidfile=${OPENSHIFT_CELERY_DIR}etc/celeryd.pid --logfile=${OPENSHIFT_APP_ROOT}app-root/logs/celery_log.txt + echo "Starting Celery, cmd: ${OPENSHIFT_CELERY_DIR}usr/celery multi start worker --config=${OPENSHIFT_CELERY_CONFIG=celeryconfig} --loglevel=DEBUG --hostname=$OPENSHIFT_APP_DNS --pidfile=${OPENSHIFT_CELERY_DIR}etc/celeryd.pid --logfile=${OPENSHIFT_HOMEDIR}app-root/logs/celery_log.txt" + echo `${OPENSHIFT_CELERY_DIR}usr/celery multi start worker --config=${OPENSHIFT_CELERY_CONFIG=celeryconfig} --loglevel=DEBUG --hostname=$OPENSHIFT_APP_DNS --pidfile=${OPENSHIFT_CELERY_DIR}etc/celeryd.pid --logfile=${OPENSHIFT_HOMEDIR}app-root/logs/celery_log.txt` > ${OPENSHIFT_APP_ROOT}app-root/logs/celery_log.txt + ${OPENSHIFT_CELERY_DIR}usr/celery multi start worker --config=${OPENSHIFT_CELERY_CONFIG=celeryconfig} --loglevel=DEBUG --hostname=$OPENSHIFT_APP_DNS --pidfile=${OPENSHIFT_CELERY_DIR}etc/celeryd.pid --logfile=${OPENSHIFT_HOMEDIR}app-root/logs/celery_log.txt } function stop { From 2cd0c80fe3ee030cc3fcc75749f3a0abb110bb08 Mon Sep 17 00:00:00 2001 From: Is Isilon Date: Sat, 11 Apr 2015 13:08:53 +0800 Subject: [PATCH 75/85] finishing --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 9cd565f..ed2e78a 100644 --- a/README.md +++ b/README.md @@ -11,12 +11,12 @@ Configuration --------------------- * OPENSHIFT_CELERY_CONFIG - This is the name of your config file, which by default is $OPENSHIFT_CELERY_DIR/conf.d/celeryconfig.py but you can copy this to $OPENSHIFT_DATA_DIR/config/celeryconfig.py and edit your own copy. + This is the name of your config file, which by default is `$OPENSHIFT_CELERY_DIR/conf.d/celeryconfig.py` but you can copy this to `$OPENSHIFT_DATA_DIR/config/celeryconfig.py` and edit your own copy. To install --------------------- - rhc cartridge-add https://raw.github.com/wassname/openshift-celery-cartridge/master/metadata/manifest.yml -a "appname" + rhc cartridge-add https://raw.github.com/wassname/openshift-celery-cartridge/master/metadata/manifest.yml -a `` Any log output will be generated to ${OPENSHIFT_HOMEDIR}logs/celery_log.txt and will be viewable with the rhc tail "appname" command From 8b408d4e55b1047522b4275f4a157f8bf8bb6720 Mon Sep 17 00:00:00 2001 From: Is Isilon Date: Mon, 13 Apr 2015 13:15:52 +0800 Subject: [PATCH 76/85] 1 --- README.md | 4 ++-- bin/control | 10 ++++++++-- metadata/manifest.yml | 5 +++-- 3 files changed, 13 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index ed2e78a..3d25973 100644 --- a/README.md +++ b/README.md @@ -16,9 +16,9 @@ Configuration To install --------------------- - rhc cartridge-add https://raw.github.com/wassname/openshift-celery-cartridge/master/metadata/manifest.yml -a `` + rhc cartridge-add https://raw.github.com/wassname/openshift-celery-cartridge/master/metadata/manifest.yml -a "appname" -Any log output will be generated to ${OPENSHIFT_HOMEDIR}logs/celery_log.txt and will be viewable with the rhc tail "appname" command +Any log output will be generated to `${OPENSHIFT_HOMEDIR}logs/celery_log.txt` and will be viewable with the rhc tail "appname" command This was tested using: diff --git a/bin/control b/bin/control index b368495..b556cc1 100755 --- a/bin/control +++ b/bin/control @@ -1,8 +1,13 @@ #!/bin/bash -e source $OPENSHIFT_CARTRIDGE_SDK_BASH - +source $OPENSHIFT_PYTHON_DIRvirtenv/bin/activate PATH=$OPENSHIFT_DATA_DIR:$OPENSHIFT_DATA_DIR/config/:${OPENSHIFT_CELERY_DIR}bin/:${OPENSHIFT_CELERY_DIR}usr/bin:${OPENSHIFT_CELERY_DIR}conf.d:$PATH +echo find . +echo $OPENSHIFT_PYTHON_DIR +echo ${OPENSHIFT_CELERY_DIR} +echo var + #TODO add scaling, better status, variable worker names, stop workers properly @@ -10,7 +15,7 @@ function start { export PYTHONPATH=$OPENSHIFT_DATA_DIR:$OPENSHIFT_DATA_DIR/config/:$OPENSHIFT_REPO_DIR/.openshift:$OPENSHIFT_REPO_DIR/wsgi::$OPENSHIFT_CELERY_DIR/conf.d:$OPENSHIFT_REPO_DIR:$PYTHONPATH PYTHONPATH=$OPENSHIFT_DATA_DIR:$OPENSHIFT_DATA_DIR/config/:$OPENSHIFT_REPO_DIR/.openshift:$OPENSHIFT_REPO_DIR/wsgi::$OPENSHIFT_CELERY_DIR/conf.d:$OPENSHIFT_REPO_DIR:$PYTHONPATH echo "Starting Celery, cmd: ${OPENSHIFT_CELERY_DIR}usr/celery multi start worker --config=${OPENSHIFT_CELERY_CONFIG=celeryconfig} --loglevel=DEBUG --hostname=$OPENSHIFT_APP_DNS --pidfile=${OPENSHIFT_CELERY_DIR}etc/celeryd.pid --logfile=${OPENSHIFT_HOMEDIR}app-root/logs/celery_log.txt" - echo `${OPENSHIFT_CELERY_DIR}usr/celery multi start worker --config=${OPENSHIFT_CELERY_CONFIG=celeryconfig} --loglevel=DEBUG --hostname=$OPENSHIFT_APP_DNS --pidfile=${OPENSHIFT_CELERY_DIR}etc/celeryd.pid --logfile=${OPENSHIFT_HOMEDIR}app-root/logs/celery_log.txt` > ${OPENSHIFT_APP_ROOT}app-root/logs/celery_log.txt + #echo `${OPENSHIFT_CELERY_DIR}usr/celery multi start worker --config=${OPENSHIFT_CELERY_CONFIG=celeryconfig} --loglevel=DEBUG --hostname=$OPENSHIFT_APP_DNS --pidfile=${OPENSHIFT_CELERY_DIR}etc/celeryd.pid --logfile=${OPENSHIFT_HOMEDIR}app-root/logs/celery_log.txt` > ${OPENSHIFT_APP_ROOT}app-root/logs/celery_log.txt ${OPENSHIFT_CELERY_DIR}usr/celery multi start worker --config=${OPENSHIFT_CELERY_CONFIG=celeryconfig} --loglevel=DEBUG --hostname=$OPENSHIFT_APP_DNS --pidfile=${OPENSHIFT_CELERY_DIR}etc/celeryd.pid --logfile=${OPENSHIFT_HOMEDIR}app-root/logs/celery_log.txt } @@ -22,6 +27,7 @@ function stop { echo "nothing to kill" fi echo "stopped" + rm -f ${OPENSHIFT_CELERY_DIR}etc/celeryd.pid } function restart { diff --git a/metadata/manifest.yml b/metadata/manifest.yml index 633bec2..8e0ca37 100644 --- a/metadata/manifest.yml +++ b/metadata/manifest.yml @@ -13,6 +13,7 @@ Source-Url: https://github.com/wassname/openshift-celery-cartridge.git Categories: - service - embedded + - python Requires: celery Scaling: Min: 1 @@ -38,11 +39,11 @@ Group-Overrides: - components: - celery Subscribes: - set-redis-connection-info: + set-rcelery-connection-info: Type: "ENV:NET_TCP:db:celery:connection-info-v1" Required: false Publishes: publish-db-connection-info: Type: "ENV:NET_TCP:db:connection-info" - publish-redis-connection-info: + publish-celery-connection-info: Type: "ENV:NET_TCP:db:celery:connection-info-v1" From 96278f9ff355bb5c4be2e515a7e3c2afc8291f10 Mon Sep 17 00:00:00 2001 From: Is Isilon Date: Mon, 13 Apr 2015 13:16:54 +0800 Subject: [PATCH 77/85] 1 --- bin/control | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/bin/control b/bin/control index b556cc1..26e6db7 100755 --- a/bin/control +++ b/bin/control @@ -3,10 +3,10 @@ source $OPENSHIFT_CARTRIDGE_SDK_BASH source $OPENSHIFT_PYTHON_DIRvirtenv/bin/activate PATH=$OPENSHIFT_DATA_DIR:$OPENSHIFT_DATA_DIR/config/:${OPENSHIFT_CELERY_DIR}bin/:${OPENSHIFT_CELERY_DIR}usr/bin:${OPENSHIFT_CELERY_DIR}conf.d:$PATH -echo find . -echo $OPENSHIFT_PYTHON_DIR -echo ${OPENSHIFT_CELERY_DIR} -echo var +echo find . > ${OPENSHIFT_HOMEDIR}app-root/logs/celery_log.txt +echo $OPENSHIFT_PYTHON_DIR >> ${OPENSHIFT_HOMEDIR}app-root/logs/celery_log.txt +echo ${OPENSHIFT_CELERY_DIR} >> ${OPENSHIFT_HOMEDIR}app-root/logs/celery_log.txt +echo var >> ${OPENSHIFT_HOMEDIR}app-root/logs/celery_log.txt #TODO add scaling, better status, variable worker names, stop workers properly From afe22e82f78b2edb086c53ddf07dd5aeb0cccc96 Mon Sep 17 00:00:00 2001 From: Is Isilon Date: Mon, 13 Apr 2015 13:32:41 +0800 Subject: [PATCH 78/85] 2 --- metadata/manifest.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/metadata/manifest.yml b/metadata/manifest.yml index 8e0ca37..675be4f 100644 --- a/metadata/manifest.yml +++ b/metadata/manifest.yml @@ -11,7 +11,7 @@ Cartridge-Vendor: wassname Website: https://github.com/wassname/openshift-celery-cartridge Source-Url: https://github.com/wassname/openshift-celery-cartridge.git Categories: - - service + - plugin - embedded - python Requires: celery @@ -38,6 +38,7 @@ Cart-Data: Group-Overrides: - components: - celery + - python Subscribes: set-rcelery-connection-info: Type: "ENV:NET_TCP:db:celery:connection-info-v1" From d032b711febc2223b356cac78ff424a8cabdf97f Mon Sep 17 00:00:00 2001 From: Is Isilon Date: Mon, 13 Apr 2015 13:36:06 +0800 Subject: [PATCH 79/85] 2 --- metadata/manifest.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/metadata/manifest.yml b/metadata/manifest.yml index 675be4f..aab7c58 100644 --- a/metadata/manifest.yml +++ b/metadata/manifest.yml @@ -13,8 +13,7 @@ Source-Url: https://github.com/wassname/openshift-celery-cartridge.git Categories: - plugin - embedded - - python -Requires: celery +Requires: python Scaling: Min: 1 Max: -1 From 3f27c8b5fb70faeca924d7d9a7be39ede2a871db Mon Sep 17 00:00:00 2001 From: Is Isilon Date: Mon, 13 Apr 2015 13:37:23 +0800 Subject: [PATCH 80/85] 2 --- bin/control | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/bin/control b/bin/control index 26e6db7..e152400 100755 --- a/bin/control +++ b/bin/control @@ -1,12 +1,12 @@ #!/bin/bash -e source $OPENSHIFT_CARTRIDGE_SDK_BASH -source $OPENSHIFT_PYTHON_DIRvirtenv/bin/activate +#source $OPENSHIFT_PYTHON_DIRvirtenv/bin/activate PATH=$OPENSHIFT_DATA_DIR:$OPENSHIFT_DATA_DIR/config/:${OPENSHIFT_CELERY_DIR}bin/:${OPENSHIFT_CELERY_DIR}usr/bin:${OPENSHIFT_CELERY_DIR}conf.d:$PATH -echo find . > ${OPENSHIFT_HOMEDIR}app-root/logs/celery_log.txt -echo $OPENSHIFT_PYTHON_DIR >> ${OPENSHIFT_HOMEDIR}app-root/logs/celery_log.txt -echo ${OPENSHIFT_CELERY_DIR} >> ${OPENSHIFT_HOMEDIR}app-root/logs/celery_log.txt -echo var >> ${OPENSHIFT_HOMEDIR}app-root/logs/celery_log.txt +#echo find . > ${OPENSHIFT_HOMEDIR}app-root/logs/celery_log.txt +#echo $OPENSHIFT_PYTHON_DIR >> ${OPENSHIFT_HOMEDIR}app-root/logs/celery_log.txt +#echo ${OPENSHIFT_CELERY_DIR} >> ${OPENSHIFT_HOMEDIR}app-root/logs/celery_log.txt +#echo var >> ${OPENSHIFT_HOMEDIR}app-root/logs/celery_log.txt #TODO add scaling, better status, variable worker names, stop workers properly From d702eecb5524d4af4b82e59c7fe670709e475a76 Mon Sep 17 00:00:00 2001 From: Is Isilon Date: Mon, 13 Apr 2015 13:41:05 +0800 Subject: [PATCH 81/85] 3 --- bin/control | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/bin/control b/bin/control index e152400..fb25021 100755 --- a/bin/control +++ b/bin/control @@ -12,11 +12,14 @@ PATH=$OPENSHIFT_DATA_DIR:$OPENSHIFT_DATA_DIR/config/:${OPENSHIFT_CELERY_DIR}bin/ #TODO add scaling, better status, variable worker names, stop workers properly function start { + LOGPIPE=${OPENSHIFT_HOMEDIR}/app-root/runtime/logshifter-python + rm -f $LOGPIPE && mkfifo $LOGPIPE + /usr/bin/logshifter -tag python < $LOGPIPE & export PYTHONPATH=$OPENSHIFT_DATA_DIR:$OPENSHIFT_DATA_DIR/config/:$OPENSHIFT_REPO_DIR/.openshift:$OPENSHIFT_REPO_DIR/wsgi::$OPENSHIFT_CELERY_DIR/conf.d:$OPENSHIFT_REPO_DIR:$PYTHONPATH PYTHONPATH=$OPENSHIFT_DATA_DIR:$OPENSHIFT_DATA_DIR/config/:$OPENSHIFT_REPO_DIR/.openshift:$OPENSHIFT_REPO_DIR/wsgi::$OPENSHIFT_CELERY_DIR/conf.d:$OPENSHIFT_REPO_DIR:$PYTHONPATH echo "Starting Celery, cmd: ${OPENSHIFT_CELERY_DIR}usr/celery multi start worker --config=${OPENSHIFT_CELERY_CONFIG=celeryconfig} --loglevel=DEBUG --hostname=$OPENSHIFT_APP_DNS --pidfile=${OPENSHIFT_CELERY_DIR}etc/celeryd.pid --logfile=${OPENSHIFT_HOMEDIR}app-root/logs/celery_log.txt" #echo `${OPENSHIFT_CELERY_DIR}usr/celery multi start worker --config=${OPENSHIFT_CELERY_CONFIG=celeryconfig} --loglevel=DEBUG --hostname=$OPENSHIFT_APP_DNS --pidfile=${OPENSHIFT_CELERY_DIR}etc/celeryd.pid --logfile=${OPENSHIFT_HOMEDIR}app-root/logs/celery_log.txt` > ${OPENSHIFT_APP_ROOT}app-root/logs/celery_log.txt - ${OPENSHIFT_CELERY_DIR}usr/celery multi start worker --config=${OPENSHIFT_CELERY_CONFIG=celeryconfig} --loglevel=DEBUG --hostname=$OPENSHIFT_APP_DNS --pidfile=${OPENSHIFT_CELERY_DIR}etc/celeryd.pid --logfile=${OPENSHIFT_HOMEDIR}app-root/logs/celery_log.txt + nohup ${OPENSHIFT_CELERY_DIR}usr/celery multi start worker --config=${OPENSHIFT_CELERY_CONFIG=celeryconfig} --loglevel=DEBUG --hostname=$OPENSHIFT_APP_DNS --pidfile=${OPENSHIFT_CELERY_DIR}etc/celeryd.pid --logfile=${OPENSHIFT_HOMEDIR}app-root/logs/celery_log.txt &> $LOGPIPE & } function stop { From f65a44cb0f377b7d080317a90a790a4a86c8e211 Mon Sep 17 00:00:00 2001 From: Is Isilon Date: Mon, 13 Apr 2015 14:14:33 +0800 Subject: [PATCH 82/85] 3 --- bin/control | 6 ++++++ metadata/manifest.yml | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/bin/control b/bin/control index fb25021..3e32281 100755 --- a/bin/control +++ b/bin/control @@ -20,6 +20,12 @@ function start { echo "Starting Celery, cmd: ${OPENSHIFT_CELERY_DIR}usr/celery multi start worker --config=${OPENSHIFT_CELERY_CONFIG=celeryconfig} --loglevel=DEBUG --hostname=$OPENSHIFT_APP_DNS --pidfile=${OPENSHIFT_CELERY_DIR}etc/celeryd.pid --logfile=${OPENSHIFT_HOMEDIR}app-root/logs/celery_log.txt" #echo `${OPENSHIFT_CELERY_DIR}usr/celery multi start worker --config=${OPENSHIFT_CELERY_CONFIG=celeryconfig} --loglevel=DEBUG --hostname=$OPENSHIFT_APP_DNS --pidfile=${OPENSHIFT_CELERY_DIR}etc/celeryd.pid --logfile=${OPENSHIFT_HOMEDIR}app-root/logs/celery_log.txt` > ${OPENSHIFT_APP_ROOT}app-root/logs/celery_log.txt nohup ${OPENSHIFT_CELERY_DIR}usr/celery multi start worker --config=${OPENSHIFT_CELERY_CONFIG=celeryconfig} --loglevel=DEBUG --hostname=$OPENSHIFT_APP_DNS --pidfile=${OPENSHIFT_CELERY_DIR}etc/celeryd.pid --logfile=${OPENSHIFT_HOMEDIR}app-root/logs/celery_log.txt &> $LOGPIPE & + if ps -p `cat ${OPENSHIFT_CELERY_DIR}etc/celeryd.pid` > /dev/null; + then + echo "Startign celery worked" + else + echo "Startign celery failed" + fi } function stop { diff --git a/metadata/manifest.yml b/metadata/manifest.yml index aab7c58..6013921 100644 --- a/metadata/manifest.yml +++ b/metadata/manifest.yml @@ -11,7 +11,7 @@ Cartridge-Vendor: wassname Website: https://github.com/wassname/openshift-celery-cartridge Source-Url: https://github.com/wassname/openshift-celery-cartridge.git Categories: - - plugin + - service - embedded Requires: python Scaling: From 37b15a9d1920c4c7001e49b6938a660d4e0ae10f Mon Sep 17 00:00:00 2001 From: Is Isilon Date: Mon, 13 Apr 2015 14:15:18 +0800 Subject: [PATCH 83/85] 3 --- metadata/manifest.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metadata/manifest.yml b/metadata/manifest.yml index 6013921..aab7c58 100644 --- a/metadata/manifest.yml +++ b/metadata/manifest.yml @@ -11,7 +11,7 @@ Cartridge-Vendor: wassname Website: https://github.com/wassname/openshift-celery-cartridge Source-Url: https://github.com/wassname/openshift-celery-cartridge.git Categories: - - service + - plugin - embedded Requires: python Scaling: From e5dcb571a6de91820637c9f7ec0d1422e06ebd44 Mon Sep 17 00:00:00 2001 From: Is Isilon Date: Mon, 20 Apr 2015 17:06:13 +1200 Subject: [PATCH 84/85] Bugfix --- README.md | 4 +++- bin/control | 6 +++--- conf.d/celeryconfig.py | 2 +- 3 files changed, 7 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 3d25973..24e2a64 100644 --- a/README.md +++ b/README.md @@ -5,7 +5,9 @@ Cartridge to Expose Celery as a Daemon on OpenShift. Status --------------------- -This work on unscaled applications but doesn't work as a gearin a scaled application. Further development can combine this with a python cartridge so it can operate in it's own gears. +This work on unscaled applications but only as a plugin cartridge on scaled applications. This means that it's can't scale independently of your main gear. At this point having a celery cartridge is no better than just starting celery in your app.y or action hooks. Further development can combine this with a python cartridge so it can operate in it's own gears. + +Instead of using this I recommend starting celery in your action hooks or as part of you app.py instead. This will give more flexibility, and example is here https://github.com/appsembler/appsembler-launch-openshift Configuration --------------------- diff --git a/bin/control b/bin/control index 3e32281..1e02bd5 100755 --- a/bin/control +++ b/bin/control @@ -15,14 +15,14 @@ function start { LOGPIPE=${OPENSHIFT_HOMEDIR}/app-root/runtime/logshifter-python rm -f $LOGPIPE && mkfifo $LOGPIPE /usr/bin/logshifter -tag python < $LOGPIPE & - export PYTHONPATH=$OPENSHIFT_DATA_DIR:$OPENSHIFT_DATA_DIR/config/:$OPENSHIFT_REPO_DIR/.openshift:$OPENSHIFT_REPO_DIR/wsgi::$OPENSHIFT_CELERY_DIR/conf.d:$OPENSHIFT_REPO_DIR:$PYTHONPATH - PYTHONPATH=$OPENSHIFT_DATA_DIR:$OPENSHIFT_DATA_DIR/config/:$OPENSHIFT_REPO_DIR/.openshift:$OPENSHIFT_REPO_DIR/wsgi::$OPENSHIFT_CELERY_DIR/conf.d:$OPENSHIFT_REPO_DIR:$PYTHONPATH + export PYTHONPATH=$OPENSHIFT_DATA_DIR:$OPENSHIFT_DATA_DIR/config/:$OPENSHIFT_REPO_DIR/.openshift:$OPENSHIFT_REPO_DIR/config:$OPENSHIFT_REPO_DIR/EXT:$OPENSHIFT_REPO_DIR/wsgi::$OPENSHIFT_CELERY_DIR/conf.d:$OPENSHIFT_REPO_DIR:$PYTHONPATH + PYTHONPATH=$OPENSHIFT_DATA_DIR:$OPENSHIFT_DATA_DIR/config/:$OPENSHIFT_REPO_DIR/.openshift:$OPENSHIFT_REPO_DIR/config:$OPENSHIFT_REPO_DIR/EXT:$OPENSHIFT_REPO_DIR/wsgi::$OPENSHIFT_CELERY_DIR/conf.d:$OPENSHIFT_REPO_DIR:$PYTHONPATH echo "Starting Celery, cmd: ${OPENSHIFT_CELERY_DIR}usr/celery multi start worker --config=${OPENSHIFT_CELERY_CONFIG=celeryconfig} --loglevel=DEBUG --hostname=$OPENSHIFT_APP_DNS --pidfile=${OPENSHIFT_CELERY_DIR}etc/celeryd.pid --logfile=${OPENSHIFT_HOMEDIR}app-root/logs/celery_log.txt" #echo `${OPENSHIFT_CELERY_DIR}usr/celery multi start worker --config=${OPENSHIFT_CELERY_CONFIG=celeryconfig} --loglevel=DEBUG --hostname=$OPENSHIFT_APP_DNS --pidfile=${OPENSHIFT_CELERY_DIR}etc/celeryd.pid --logfile=${OPENSHIFT_HOMEDIR}app-root/logs/celery_log.txt` > ${OPENSHIFT_APP_ROOT}app-root/logs/celery_log.txt nohup ${OPENSHIFT_CELERY_DIR}usr/celery multi start worker --config=${OPENSHIFT_CELERY_CONFIG=celeryconfig} --loglevel=DEBUG --hostname=$OPENSHIFT_APP_DNS --pidfile=${OPENSHIFT_CELERY_DIR}etc/celeryd.pid --logfile=${OPENSHIFT_HOMEDIR}app-root/logs/celery_log.txt &> $LOGPIPE & if ps -p `cat ${OPENSHIFT_CELERY_DIR}etc/celeryd.pid` > /dev/null; then - echo "Startign celery worked" + echo "Starting celery worked" else echo "Startign celery failed" fi diff --git a/conf.d/celeryconfig.py b/conf.d/celeryconfig.py index daae2e1..ac04e6d 100644 --- a/conf.d/celeryconfig.py +++ b/conf.d/celeryconfig.py @@ -13,7 +13,7 @@ BROKER_USER = "celeryuser" BROKER_PASSWORD = "celery" BROKER_VHOST = "celeryvhost" - +CELERYD_OPTS="--autoscale=3,1 --time-limit=300 --concurrency=2 " CELERY_RESULT_BACKEND = "amqp" # Here we check the env vars first, otherwise go with "tasks" From 5659011623bbede5b5ac0d6df21512ff03b6d85b Mon Sep 17 00:00:00 2001 From: Is Isilon Date: Tue, 21 Apr 2015 09:28:46 +1200 Subject: [PATCH 85/85] Bugfixes --- bin/control | 33 +++++++++++++-------------------- 1 file changed, 13 insertions(+), 20 deletions(-) diff --git a/bin/control b/bin/control index 1e02bd5..011b369 100755 --- a/bin/control +++ b/bin/control @@ -1,56 +1,49 @@ #!/bin/bash -e source $OPENSHIFT_CARTRIDGE_SDK_BASH -#source $OPENSHIFT_PYTHON_DIRvirtenv/bin/activate -PATH=$OPENSHIFT_DATA_DIR:$OPENSHIFT_DATA_DIR/config/:${OPENSHIFT_CELERY_DIR}bin/:${OPENSHIFT_CELERY_DIR}usr/bin:${OPENSHIFT_CELERY_DIR}conf.d:$PATH -#echo find . > ${OPENSHIFT_HOMEDIR}app-root/logs/celery_log.txt -#echo $OPENSHIFT_PYTHON_DIR >> ${OPENSHIFT_HOMEDIR}app-root/logs/celery_log.txt -#echo ${OPENSHIFT_CELERY_DIR} >> ${OPENSHIFT_HOMEDIR}app-root/logs/celery_log.txt -#echo var >> ${OPENSHIFT_HOMEDIR}app-root/logs/celery_log.txt - - -#TODO add scaling, better status, variable worker names, stop workers properly function start { LOGPIPE=${OPENSHIFT_HOMEDIR}/app-root/runtime/logshifter-python rm -f $LOGPIPE && mkfifo $LOGPIPE /usr/bin/logshifter -tag python < $LOGPIPE & - export PYTHONPATH=$OPENSHIFT_DATA_DIR:$OPENSHIFT_DATA_DIR/config/:$OPENSHIFT_REPO_DIR/.openshift:$OPENSHIFT_REPO_DIR/config:$OPENSHIFT_REPO_DIR/EXT:$OPENSHIFT_REPO_DIR/wsgi::$OPENSHIFT_CELERY_DIR/conf.d:$OPENSHIFT_REPO_DIR:$PYTHONPATH - PYTHONPATH=$OPENSHIFT_DATA_DIR:$OPENSHIFT_DATA_DIR/config/:$OPENSHIFT_REPO_DIR/.openshift:$OPENSHIFT_REPO_DIR/config:$OPENSHIFT_REPO_DIR/EXT:$OPENSHIFT_REPO_DIR/wsgi::$OPENSHIFT_CELERY_DIR/conf.d:$OPENSHIFT_REPO_DIR:$PYTHONPATH - echo "Starting Celery, cmd: ${OPENSHIFT_CELERY_DIR}usr/celery multi start worker --config=${OPENSHIFT_CELERY_CONFIG=celeryconfig} --loglevel=DEBUG --hostname=$OPENSHIFT_APP_DNS --pidfile=${OPENSHIFT_CELERY_DIR}etc/celeryd.pid --logfile=${OPENSHIFT_HOMEDIR}app-root/logs/celery_log.txt" - #echo `${OPENSHIFT_CELERY_DIR}usr/celery multi start worker --config=${OPENSHIFT_CELERY_CONFIG=celeryconfig} --loglevel=DEBUG --hostname=$OPENSHIFT_APP_DNS --pidfile=${OPENSHIFT_CELERY_DIR}etc/celeryd.pid --logfile=${OPENSHIFT_HOMEDIR}app-root/logs/celery_log.txt` > ${OPENSHIFT_APP_ROOT}app-root/logs/celery_log.txt - nohup ${OPENSHIFT_CELERY_DIR}usr/celery multi start worker --config=${OPENSHIFT_CELERY_CONFIG=celeryconfig} --loglevel=DEBUG --hostname=$OPENSHIFT_APP_DNS --pidfile=${OPENSHIFT_CELERY_DIR}etc/celeryd.pid --logfile=${OPENSHIFT_HOMEDIR}app-root/logs/celery_log.txt &> $LOGPIPE & + + PYTHONPATH=$OPENSHIFT_DATA_DIR:$OPENSHIFT_DATA_DIR/config/:$OPENSHIFT_REPO_DIR/.openshift:$OPENSHIFT_REPO_DIR/config:$OPENSHIFT_REPO_DIR/EXT:$OPENSHIFT_REPO_DIR/wsgi::$OPENSHIFT_CELERY_DIR/conf.d:$OPENSHIFT_REPO_DIR:$PYTHONPATH + + echo "Starting Celery" + #echo "nohup nice -n 10 ${OPENSHIFT_CELERY_DIR}usr/celery multi start worker --config=${OPENSHIFT_CELERY_CONFIG=celeryconfig} --loglevel=DEBUG --hostname=$OPENSHIFT_APP_DNS --pidfile=${OPENSHIFT_CELERY_DIR}etc/celeryd.pid --logfile=${OPENSHIFT_HOMEDIR}app-root/logs/celery_log.txt &> $LOGPIPE &" > ${OPENSHIFT_LOG_DIR}celery_log.txt + nohup nice -n 10 ${OPENSHIFT_CELERY_DIR}usr/celery multi start worker --config=${OPENSHIFT_CELERY_CONFIG=celeryconfig} --loglevel=DEBUG --hostname=$OPENSHIFT_APP_DNS --pidfile=${OPENSHIFT_CELERY_DIR}etc/celeryd.pid --logfile=${OPENSHIFT_LOG_DIR}celery_log.txt &> $LOGPIPE & if ps -p `cat ${OPENSHIFT_CELERY_DIR}etc/celeryd.pid` > /dev/null; then echo "Starting celery worked" else - echo "Startign celery failed" + echo "Starting celery failed" fi } function stop { + celery multi stop worker --pidfile=${OPENSHIFT_CELERY_DIR}etc/celeryd.pid if ps -p `cat ${OPENSHIFT_CELERY_DIR}etc/celeryd.pid` > /dev/null; then kill -9 `cat ${OPENSHIFT_CELERY_DIR}etc/celeryd.pid` + rm -f ${OPENSHIFT_CELERY_DIR}etc/celeryd.pid else echo "nothing to kill" fi echo "stopped" - rm -f ${OPENSHIFT_CELERY_DIR}etc/celeryd.pid } function restart { - celery multi restart worker --pidfile=${OPENSHIFT_CELERY_DIR}etc/celeryd.pid - echo "not yet implemented" + celery multi restart worker --config=${OPENSHIFT_CELERY_CONFIG=celeryconfig} --loglevel=DEBUG --hostname=$OPENSHIFT_APP_DNS --pidfile=${OPENSHIFT_CELERY_DIR}etc/celeryd.pid --logfile=${OPENSHIFT_LOG_DIR}celery_log.txt + echo "Restarted celery" } function status() { pid=`cat ${OPENSHIFT_CELERY_DIR}etc/celeryd.pid` - info=`celery multi show worker` + PROCESS_COUNT=$(ps -ef | grep "${PROCESS_NAME}" | grep -v "grep" | wc -l) client_result $pwd if [ -f $pid ] && ( kill -0 $(cat $pid) ); then - client_result "Celery is running ${info} at pid:${pid}" + client_result "Celery is running at pid:${pid}, $PROCESS_COUNT instances of celery" else client_result "Celery is not running" fi