Merge branch 'master' into feature/autofocus

Conflicts:
	ide/static/ide/js/resources.js
	ide/templates/ide/project/resource.html

Author: Joseph Atkins-Turkish

.gitignore

@@ -11,3 +11,4 @@ src/
 .cache/
 .env/
 .idea/
+bower_components

Dockerfile

@@ -0,0 +1,71 @@
+FROM python:2.7
+MAINTAINER Katharine Berry <[email protected]>
+
+ENV NPM_CONFIG_LOGLEVEL=info NODE_VERSION=4.2.3 DJANGO_VERSION=1.6
+
+# Node stuff.
+
+# gpg keys listed at https://github.com/nodejs/node
+RUN set -ex \
+  && for key in \
+    9554F04D7259F04124DE6B476D5A82AC7E37093B \
+    94AE36675C464D64BAFA68DD7434390BDBE9B9C5 \
+    0034A06D9D9B0064CE8ADF6BF1747F4AD2306D93 \
+    FD3A5288F042B6850C66B31F09FE44734EB7990E \
+    71DCFD284A79C3B38668286BC97EC7A07EDE3FC1 \
+    DD8F2338BAE7501E3DD5AC78C273792F7D83545D \
+  ; do \
+    gpg --keyserver ha.pool.sks-keyservers.net --recv-keys "$key"; \
+  done
+
+RUN curl -SLO "https://nodejs.org/dist/v$NODE_VERSION/node-v$NODE_VERSION-linux-x64.tar.gz" \
+  && curl -SLO "https://nodejs.org/dist/v$NODE_VERSION/SHASUMS256.txt.asc" \
+  && gpg --verify SHASUMS256.txt.asc \
+  && grep " node-v$NODE_VERSION-linux-x64.tar.gz\$" SHASUMS256.txt.asc | sha256sum -c - \
+  && tar -xzf "node-v$NODE_VERSION-linux-x64.tar.gz" -C /usr/local --strip-components=1 \
+  && rm "node-v$NODE_VERSION-linux-x64.tar.gz" SHASUMS256.txt.asc
+
+# Django stuff
+
+RUN apt-get update && apt-get install -y \
+    gettext \
+    postgresql-client libpq-dev \
+  --no-install-recommends && rm -rf /var/lib/apt/lists/*
+
+RUN pip install psycopg2 django=="$DJANGO_VERSION"
+
+EXPOSE 8000
+
+# CloudPebble stuff
+RUN npm install -g bower && echo '{"allow_root": true}' > ~/.bowerrc
+
+# Grab the toolchain
+RUN curl -o /tmp/arm-cs-tools.tar https://cloudpebble-vagrant.s3.amazonaws.com/arm-cs-tools-stripped.tar && \
+  tar -xf /tmp/arm-cs-tools.tar -C / && rm /tmp/arm-cs-tools.tar
+
+ADD requirements.txt /tmp/requirements.txt
+RUN pip install --no-cache-dir -r /tmp/requirements.txt
+
+ENV SDK_TWO_VERSION=2.9
+
+# Install SDK 2
+RUN mkdir /sdk2 && \
+  curl -L "https://s3.amazonaws.com/assets.getpebble.com/sdk3/sdk-core/sdk-core-${SDK_TWO_VERSION}.tar.bz2" | \
+  tar --strip-components=1 -xj -C /sdk2
+
+ENV SDK_THREE_VERSION=3.8.1
+
+# Install SDK 3
+RUN mkdir /sdk3 && \
+  curl -L "https://s3.amazonaws.com/assets.getpebble.com/sdk3/release/sdk-core-${SDK_THREE_VERSION}.tar.bz2" | \
+  tar --strip-components=1 -xj -C /sdk3
+
+COPY . /code
+WORKDIR /code
+
+# Bower is awful.
+RUN rm -rf bower_components && cd /tmp && python /code/manage.py bower install && mv bower_components /code/
+
+RUN python manage.py compilemessages
+
+CMD ["sh", "docker_start.sh"]

README.md

@@ -7,30 +7,12 @@ support.
 Getting Started
 ---------------
 
-The easiest way to get an instance of CloudPebble running is to use [Vagrant][].
-The supplied virtual machine assumes [VirtualBox][], so you'll want to have that installed too.
-You will also need ports 8000 and 8001 to be available on your machine.
+The easiest way to get a fully-functional copy of CloudPebble is using Docker and Docker Compose. Instructions can
+be found in the [https://github.com/pebble/cloudpebble-composed](cloudpebble-composed) repo.
 
-With those set up, just clone the repo and run `vagrant up`:
 
-    host:~/cloudpebble$ vagrant up
-
-After some waiting, you should have a functional instance running at [http://localhost:8000](http://localhost:8000).
-
-In general, you can then stop it with `vagrant halt`, start it with `vagrant up`, and restart it with `vagrant reload`.
-If you need to recreate the thing, you can use `vagrant destroy` and `vagrant up` again. No persistent data is stored
-in the VM, so doing this is safe.
-
-The server will be running in debug mode with autoreload enabled. However, the worker will not autoreload, so if you
-make changes that the worker should pick up on, you must restart it manually:
-
-
-    host:~/cloudpebble$ vagrant ssh
-    vagrant@precise32:~$ sudo restart cloudpebble-celery
-
-
-To locally override the configuration, create a file at `cloudpebble/settings_local.py` and set the appropriate values
-there.
+To locally override the configuration, you can create a file at `cloudpebble/settings_local.py` and set the
+appropriate values there. Setting environment variables also works.
 
 Note that you won't be able to set up integration with certain Pebble systems (e.g. Pebble SSO). This shouldn't usually
 matter; whenever these are used, an alternative route is provided and should be invoked in its absence.
@@ -44,6 +26,4 @@ logical sense. Please avoid commits that fix typos in prior commits.
 If a change is a significant amount of work, it would probably be worth creating an issue to discuss it first. Pull
 requests are not automatically accepted (though they usually are).
 
-[Vagrant]: http://www.vagrantup.com
-[VirtualBox]: http://virtualbox.org
 [support]: mailto:[email protected]

app.json

@@ -32,14 +32,13 @@
       "required": true
     },
     "QEMU_URLS": "https://qemu-us1.cloudpebble.net/",
-    "SDK2_PEBBLE_TOOL": "/app/sdk2/bin/pebble",
-    "SDK3_PEBBLE_TOOL": "/app/sdk3/bin/pebble",
+    "SDK2_PEBBLE_WAF": "/app/sdk2/pebble/waf",
+    "SDK3_PEBBLE_WAF": "/app/sdk3/pebble/waf",
     "TYPOGRAPHY_CSS": "//cloud.typography.com/7515512/745724/css/fonts.css",
     "WEB_CONCURRENCY": "2",
     "YCM_URLS": "https://ycm1.cloudpebble.net/"
   },
   "addons": [
-    "cloudamqp:lemur",
     "papertrail:choklad",
     "heroku-postgresql:hobby-dev",
     "heroku-redis:hobby-dev"

bin/post_compile

@@ -6,37 +6,26 @@ set -e
 # Make sure we have the SDK in place.
 
 echo "Downloading SDK 2"
-curl -o /tmp/pebblesdk2.tar.gz -L https://sdk.getpebble.com/download/2.8.1?source=cloudpebble
+curl -o /tmp/pebblesdk2.tar.gz -L https://s3.amazonaws.com/assets.getpebble.com/sdk3/sdk-core/sdk-core-2.9.tar.bz2
 
 echo "Downloading SDK 3"
-curl -o /tmp/pebblesdk3.tar.gz -L https://sdk.getpebble.com/download/3.7?source=cloudpebble
+curl -o /tmp/pebblesdk3.tar.gz -L https://s3.amazonaws.com/assets.getpebble.com/sdk3/release/sdk-core-3.8.1.tar.bz2
 
 echo "Downloading the toolchain"
 curl -o /tmp/arm-cs-tools.tar https://cloudpebble-vagrant.s3.amazonaws.com/arm-cs-tools-stripped.tar
+tar -xf /tmp/arm-cs-tools.tar
 
-echo "Extracting SDK 2"
-tar -xf /tmp/pebblesdk2.tar.gz
-mv PebbleSDK-2.8.1 sdk2
+
+mkdir sdk2
 pushd sdk2
-    echo "Extracting the toolchain"
-    tar -xf /tmp/arm-cs-tools.tar
-    echo "Preparing virtualenv"
-    virtualenv .env
-    # Instead of actually installing the requirements, we instead just use the global ones, which are a superset
-    # of the ones here.
-#    source .env/bin/activate
-#        pip install -r requirements.txt
-#    deactivate
+    echo "Extracting SDK 2"
+    tar --strip 1 -xf /tmp/pebblesdk2.tar.gz
 popd
 
 mkdir sdk3
 pushd sdk3
     echo "Extracting SDK 3"
     tar --strip 1 -xf /tmp/pebblesdk3.tar.gz
-    echo "Extracting the toolchain"
-    tar -xf /tmp/arm-cs-tools.tar
-    echo "Preparing virtualenv"
-    virtualenv .env
 popd
 
 touch NO_TRACKING

bootstrap.sh

@@ -63,7 +63,7 @@ popd
 # Obtain SDK3.
 sudo -u vagrant mkdir sdk3
 pushd sdk3
-    wget --progress=bar:force -O sdk.tar.gz https://sdk.getpebble.com/download/3.7?source=cloudpebble
+    wget --progress=bar:force -O sdk.tar.gz https://s3.amazonaws.com/assets.getpebble.com/sdk3/release/sdk-core-3.8.1.tar.bz2
     sudo -u vagrant tar --strip 1 -xzf sdk.tar.gz
     rm sdk.tar.gz
     sudo -u vagrant ln -s ~/arm-cs-tools arm-cs-tools

cloudpebble/settings.py

@@ -2,6 +2,7 @@
 # Django settings for cloudpebble project.
 
 import os
+import socket
 import dj_database_url
 _environ = os.environ
 
@@ -21,13 +22,11 @@
 if 'DATABASE_URL' not in _environ:
     DATABASES = {
         'default': {
-            'ENGINE': 'django.db.backends.sqlite3', # Add 'postgresql_psycopg2', 'mysql', 'sqlite3' or 'oracle'.
-            'NAME': os.getcwd() + '/dev.db',                      # Or path to database file if using sqlite3.
-            # The following settings are not used with sqlite3:
-            'USER': '',
-            'PASSWORD': '',
-            'HOST': '',                      # Empty for localhost through domain sockets or '127.0.0.1' for localhost through TCP.
-            'PORT': '',                      # Set to empty string for default.
+            'ENGINE': 'django.db.backends.postgresql_psycopg2',
+            'NAME': 'postgres',
+            'USER': 'postgres',
+            'HOST': 'postgres',
+            'PORT': 5432,
         }
     }
 else:
@@ -191,6 +190,12 @@
 SOCIAL_AUTH_PEBBLE_ROOT_URL = _environ.get('PEBBLE_AUTH_URL', None)
 PEBBLE_AUTH_ADMIN_TOKEN = _environ.get('PEBBLE_AUTH_ADMIN_TOKEN', None)
 
+SHOULD_BE_SECURE = _environ.get('EXPECT_SSL', '') != ''
+
+SESSION_COOKIE_HTTPONLY = True
+SESSION_COOKIE_SECURE = SHOULD_BE_SECURE
+CSRF_COOKIE_SECURE = SHOULD_BE_SECURE
+
 SOCIAL_AUTH_PEBBLE_REQUIRED = 'PEBBLE_AUTH_REQUIRED' in _environ
 
 ROOT_URLCONF = 'cloudpebble.urls'
@@ -255,7 +260,15 @@
     }
 }
 
-BROKER_URL = _environ.get('CLOUDAMQP_URL', 'amqp://guest:guest@localhost:5672/')
+REDIS_URL = _environ.get('REDIS_URL', None) or _environ.get('REDISCLOUD_URL', 'redis://redis:6379')
+
+BROKER_URL = REDIS_URL + '/1'
+CELERY_RESULT_BACKEND = BROKER_URL
+CELERY_ACCEPT_CONTENT = ['pickle']
+CELERY_TASK_SERIALIZER = 'pickle'
+
+CELERYD_TASK_TIME_LIMIT = int(_environ.get('CELERYD_TASK_TIME_LIMIT', 620))
+CELERYD_TASK_SOFT_TIME_LIMIT = int(_environ.get('CELERYD_TASK_SOFT_TIME_LIMIT', 600))
 
 BROKER_POOL_LIMIT = int(_environ.get('BROKER_POOL_LIMIT', 10))
 
@@ -278,10 +291,10 @@
 
 GITHUB_HOOK_TEMPLATE = _environ.get('GITHUB_HOOK', 'http://example.com/ide/project/%(project)d/github/push_hook?key=%(key)s')
 
-SDK2_PEBBLE_TOOL = _environ.get('SDK2_PEBBLE_TOOL', '/home/vagrant/sdk2/bin/pebble')
-SDK3_PEBBLE_TOOL = _environ.get('SDK3_PEBBLE_TOOL', '/home/vagrant/sdk3/bin/pebble')
+SDK2_PEBBLE_WAF = _environ.get('SDK2_PEBBLE_WAF', '/sdk2/pebble/waf')
+SDK3_PEBBLE_WAF = _environ.get('SDK3_PEBBLE_WAF', '/sdk3/pebble/waf')
 
-ARM_CS_TOOLS = _environ.get('ARM_CS_TOOLS', '/home/vagrant/arm-cs-tools/bin/')
+ARM_CS_TOOLS = _environ.get('ARM_CS_TOOLS', '/arm-cs-tools/bin/')
 
 KEEN_PROJECT_ID = _environ.get('KEEN_PROJECT_ID', None)
 KEEN_WRITE_KEY = _environ.get('KEEN_WRITE_KEY', None)
@@ -294,20 +307,19 @@
 AWS_ACCESS_KEY_ID = _environ.get('AWS_ACCESS_KEY_ID', None)
 AWS_SECRET_ACCESS_KEY = _environ.get('AWS_SECRET_ACCESS_KEY', None)
 
-AWS_S3_SOURCE_BUCKET = _environ.get('AWS_S3_SOURCE_BUCKET', None)
-AWS_S3_BUILDS_BUCKET = _environ.get('AWS_S3_BUILDS_BUCKET', None)
-AWS_S3_EXPORT_BUCKET = _environ.get('AWS_S3_EXPORT_BUCKET', None)
+AWS_S3_SOURCE_BUCKET = _environ.get('AWS_S3_SOURCE_BUCKET', 'source.cloudpebble.net')
+AWS_S3_BUILDS_BUCKET = _environ.get('AWS_S3_BUILDS_BUCKET', 'builds.cloudpebble.net')
+AWS_S3_EXPORT_BUCKET = _environ.get('AWS_S3_EXPORT_BUCKET', 'exports.cloudpebble.net')
+AWS_S3_FAKE_S3 = _environ.get('AWS_S3_FAKE_S3', None)
 
 TYPOGRAPHY_CSS = _environ.get('TYPOGRAPHY_CSS', None)
 
-REDIS_URL = _environ.get('REDIS_URL', None) or _environ.get('REDISCLOUD_URL', 'redis://localhost:6379/')
-
 LIBPEBBLE_PROXY = _environ.get('LIBPEBBLE_PROXY', None)
 
 YCM_URLS = _environ.get('YCM_URLS', 'http://localhost:8002/').split(',')
 COMPLETION_CERTS = _environ.get('COMPLETION_CERTS', os.getcwd() + '/completion-certs.crt')
 
-QEMU_URLS = _environ.get('QEMU_URLS', 'http://192.168.42.42:8003/').split(',')
+QEMU_URLS = _environ.get('QEMU_URLS', 'http://qemu/').split(',')
 QEMU_LAUNCH_AUTH_HEADER = _environ.get('QEMU_LAUNCH_AUTH_HEADER', 'secret')
 QEMU_LAUNCH_TIMEOUT = int(_environ.get('QEMU_LAUNCH_TIMEOUT', 15))
 
@@ -323,10 +335,12 @@
     print "No local settings overrides."
     pass
 
+socket.setdefaulttimeout(int(_environ.get("DEFAULT_SOCKET_TIMEOUT", 10)))
+
 # Don't keep these hanging around in the environment.
 if not DEBUG:
     for key in _environ.keys():
         # We need these ones to run.
-        if key in {'PATH', 'TZ', 'RUN_MAIN', 'CELERY_LOADER', 'DJANGO_SETTINGS_MODULE', 'DEBUG'}:
+        if key in {'PATH', 'TZ', 'RUN_MAIN', 'CELERY_LOADER', 'DJANGO_SETTINGS_MODULE', 'DEBUG', 'C_FORCE_ROOT'}:
             continue
         del _environ[key]

docker_start.sh

@@ -0,0 +1,16 @@
+#!/bin/sh
+sleep 1
+if [ ! -z "$RUN_WEB" ]; then
+	# Make sure the database is up to date.
+	echo "Performing database migration."
+	python manage.py syncdb --noinput
+	python manage.py migrate
+
+	python manage.py runserver 0.0.0.0:$PORT
+elif [ ! -z "$RUN_CELERY" ]; then
+	sleep 2
+	C_FORCE_ROOT=true python manage.py celery worker --autoreload --loglevel=info
+else
+	echo "Doing nothing!"
+	exit 1
+fi

ext/pebblejs

@@ -60,10 +60,10 @@ def restart_qemu_service():
 @parallel
 def update_ycmd_sdk(sdk_version):
     with cd("/home/ycm"), settings(sudo_user="ycm", shell="/bin/bash -c"):
-        sudo("wget -nv -O sdk.tar.gz https://sdk.getpebble.com/download/%s?source=cloudpebble" % sdk_version)
+        sudo("wget -nv -O sdk.tar.gz https://s3.amazonaws.com/assets.getpebble.com/sdk3/release/sdk-core-%s.tar.bz2" % sdk_version)
         sudo("tar -xf sdk.tar.gz")
         sudo("rm -rf sdk3")
-        sudo("mv PebbleSDK-%s sdk3" % sdk_version)
+        sudo("mv sdk-core sdk3")
 
 
 @task
@@ -123,6 +123,7 @@ def update_qemu_images(sdk_version):
     with lcd("~/projects/tintin"):
         local("git checkout v%s" % sdk_version)
 
+    build_qemu_image("bb2", "aplite")
     build_qemu_image("snowy_bb", "basalt")
     build_qemu_image("spalding_bb2", "chalk")
 
@@ -134,7 +135,7 @@ def update_qemu_images(sdk_version):
 @task
 @runs_once
 def update_cloudpebble_sdk(sdk_version):
-    local("sed -i.bak 's/download\/3.[a-z0-9-]*/download\/%s/' bin/post_compile bootstrap.sh" % sdk_version)
+    local("sed -i.bak 's/sdk-core-3.[a-z0-9-]*\.tar\.bz2/sdk-core-%s.tar.bz2/' bin/post_compile bootstrap.sh" % sdk_version)
     local("git add bin/post_compile bootstrap.sh")
     local("git commit -m 'Update to v%s'" % sdk_version)
     local("git push")

fabfile.py

@@ -51,7 +51,7 @@ def project_info(request, project_id):
             'file_name': x.file_name,
             'kind': x.kind,
             'identifiers': [y.resource_id for y in x.identifiers.all()],
-            'extra': {y.resource_id: {'regex': y.character_regex, 'tracking': y.tracking, 'compatibility': y.compatibility} for y in x.identifiers.all()},
+            'extra': {y.resource_id: y.get_options_dict(with_id=False) for y in x.identifiers.all()},
             'variants': [y.get_tags() for y in x.variants.all()],
         } for x in resources],
         'github': {

ide/api/project.py

@@ -24,7 +24,7 @@ def launch_emulator(request):
     oauth = request.POST['token']
     tz_offset = request.POST['tz_offset']
     versions = {
-        'aplite': '2.9',
+        'aplite': '3.0',
         'basalt': '3.0',
         'chalk': '3.0',
     }