Remove support for AWS_ENABLED==False

Author: Joseph Atkins-Turkish

app.json

@@ -7,7 +7,6 @@
     "AWS_ACCESS_KEY_ID": {
       "required": true
     },
-    "AWS_ENABLED": "yes",
     "AWS_S3_BUILDS_BUCKET": "builds-staging.cloudpebble.net",
     "AWS_S3_EXPORT_BUCKET": "export-staging.cloudpebble.net",
     "AWS_S3_SOURCE_BUCKET": "source-staging.cloudpebble.net",

cloudpebble/settings.py

@@ -302,7 +302,6 @@
 MAILCHIMP_API_KEY = _environ.get('MAILCHIMP_API_KEY', None)
 MAILCHIMP_LIST_ID = _environ.get('MAILCHIMP_LIST_ID', None)
 
-AWS_ENABLED = 'AWS_ENABLED' in _environ
 AWS_ACCESS_KEY_ID = _environ.get('AWS_ACCESS_KEY_ID', None)
 AWS_SECRET_ACCESS_KEY = _environ.get('AWS_SECRET_ACCESS_KEY', None)
 

ide/api/project.py

@@ -1,6 +1,4 @@
-import os
 import re
-import tempfile
 import time
 import json
 from django.conf import settings

ide/api/resource.py

@@ -1,5 +1,4 @@
 import json
-from django.conf import settings
 from django.contrib.auth.decorators import login_required
 from django.db import transaction
 from django.http import HttpResponse, HttpResponseRedirect
@@ -241,13 +240,8 @@ def show_resource(request, project_id, resource_id, variant):
     }
     content_disposition = "attachment; filename=\"%s\"" % resource.file_name
     content_type = content_types[resource.kind]
-    if settings.AWS_ENABLED:
-        headers = {
-            'response-content-disposition': content_disposition,
-            'Content-Type': content_type
-        }
-        return HttpResponseRedirect(s3.get_signed_url('source', variant.s3_path, headers=headers))
-    else:
-        response = HttpResponse(open(variant.local_filename), content_type=content_type)
-        response['Content-Disposition'] = content_disposition
-        return response
+    headers = {
+        'response-content-disposition': content_disposition,
+        'Content-Type': content_type
+    }
+    return HttpResponseRedirect(s3.get_signed_url('source', variant.s3_path, headers=headers))

ide/models/build.py

@@ -1,8 +1,5 @@
 import uuid
 import json
-import shutil
-import os
-import os.path
 from django.conf import settings
 from django.db import models
 from ide.models.project import Project
@@ -40,19 +37,10 @@ class BuildResult(IdeModel):
     finished = models.DateTimeField(blank=True, null=True)
 
     def _get_dir(self):
-        if settings.AWS_ENABLED:
-            return '%s/' % self.uuid
-        else:
-            path = '%s%s/%s/%s/' % (settings.MEDIA_ROOT, self.uuid[0], self.uuid[1], self.uuid)
-            if not os.path.exists(path):
-                os.makedirs(path)
-            return path
+        return '%s/' % self.uuid
 
     def get_url(self):
-        if settings.AWS_ENABLED:
-            return "%s%s/" % (settings.MEDIA_URL, self.uuid)
-        else:
-            return '%s%s/%s/%s/' % (settings.MEDIA_URL, self.uuid[0], self.uuid[1], self.uuid)
+        return "%s%s/" % (settings.MEDIA_URL, self.uuid)
 
     def get_pbw_filename(self):
         return '%swatchface.pbw' % self._get_dir()
@@ -79,39 +67,20 @@ def get_simplyjs_url(self):
         return '%ssimply.js' % self.get_url()
 
     def save_build_log(self, text):
-        if not settings.AWS_ENABLED:
-            with open(self.build_log, 'w') as f:
-                f.write(text)
-        else:
-            s3.save_file('builds', self.build_log, text, public=True, content_type='text/plain')
+        s3.save_file('builds', self.build_log, text, public=True, content_type='text/plain')
 
     def read_build_log(self):
-        if not settings.AWS_ENABLED:
-            with open(self.build_log, 'r') as f:
-                return f.read()
-        else:
-            return s3.read_file('builds', self.build_log)
+        return s3.read_file('builds', self.build_log)
 
     def save_debug_info(self, json_info, platform, kind):
         text = json.dumps(json_info)
-        if not settings.AWS_ENABLED:
-            with open(self.get_debug_info_filename(platform, kind), 'w') as f:
-                f.write(text)
-        else:
-            s3.save_file('builds', self.get_debug_info_filename(platform, kind), text, public=True, content_type='application/json')
+        s3.save_file('builds', self.get_debug_info_filename(platform, kind), text, public=True, content_type='application/json')
 
     def save_pbw(self, pbw_path):
-        if not settings.AWS_ENABLED:
-            shutil.move(pbw_path, self.pbw)
-        else:
-            s3.upload_file('builds', self.pbw, pbw_path, public=True, download_filename='%s.pbw' % self.project.app_short_name.replace('/','-'))
+        s3.upload_file('builds', self.pbw, pbw_path, public=True, download_filename='%s.pbw' % self.project.app_short_name.replace('/','-'))
 
     def save_simplyjs(self, javascript):
-        if not settings.AWS_ENABLED:
-            with open(self.simplyjs, 'w') as f:
-                f.write(javascript)
-        else:
-            s3.save_file('builds', self.simplyjs, javascript, public=True, content_type='text/javascript')
+        s3.save_file('builds', self.simplyjs, javascript, public=True, content_type='text/javascript')
 
     pbw = property(get_pbw_filename)
     build_log = property(get_build_log)

ide/models/files.py

@@ -1,10 +1,6 @@
 import os
-import shutil
 import traceback
-import datetime
 import json
-from django.conf import settings
-from django.core.validators import RegexValidator
 from django.db import models
 from django.db.models.signals import post_delete
 from django.dispatch import receiver
@@ -251,13 +247,7 @@ class Meta(IdeModel.Meta):
 @receiver(post_delete)
 def delete_file(sender, instance, **kwargs):
     if issubclass(sender, S3File):
-        if settings.AWS_ENABLED:
-            try:
-                s3.delete_file('source', instance.s3_path)
-            except:
-                traceback.print_exc()
-        else:
-            try:
-                os.unlink(instance.local_filename)
-            except OSError:
-                pass
+        try:
+            s3.delete_file('source', instance.s3_path)
+        except:
+            traceback.print_exc()

ide/models/meta.py

@@ -1,5 +1,6 @@
 from django.db import models
 
+
 class IdeModel(models.Model):
     class Meta:
         abstract = True

ide/models/s3file.py

@@ -1,6 +1,3 @@
-import shutil
-import os
-
 from django.utils.translation import ugettext as _
 from django.conf import settings
 from django.utils.timezone import now
@@ -13,7 +10,6 @@ class S3File(IdeModel):
     bucket_name = 'source'
     folder = None
     project = None
-    _create_local_if_not_exists = False
 
     @property
     def padded_id(self):
@@ -32,41 +28,11 @@ def s3_id(self):
     def s3_path(self):
         return '%s/%s' % (self.folder, self.s3_id)
 
-    def _get_contents_local(self):
-        try:
-            return open(self.local_filename).read()
-        except IOError:
-            if self._create_local_if_not_exists:
-                return ''
-            else:
-                raise
-
-    def _save_string_local(self, string):
-        if not os.path.exists(os.path.dirname(self.local_filename)):
-            os.makedirs(os.path.dirname(self.local_filename))
-        with open(self.local_filename, 'wb') as out:
-            out.write(string)
-
-    def _copy_to_path_local(self, path):
-        try:
-            shutil.copy(self.local_filename, path)
-        except IOError as err:
-            if err.errno == 2 and self._crete_local_if_not_exists:
-                open(path, 'w').close()  # create the file if it's missing.
-            else:
-                raise
-
     def get_contents(self):
-        if not settings.AWS_ENABLED:
-            return self._get_contents_local()
-        else:
-            return s3.read_file(self.bucket_name, self.s3_path)
+        return s3.read_file(self.bucket_name, self.s3_path)
 
     def save_string(self, string):
-        if not settings.AWS_ENABLED:
-            self._save_string_local(string)
-        else:
-            s3.save_file(self.bucket_name, self.s3_path, string)
+        s3.save_file(self.bucket_name, self.s3_path, string)
         if self.project:
             self.project.last_modified = now()
             self.project.save()
@@ -80,10 +46,7 @@ def save_text(self, content):
         self.save_string(content.encode('utf-8'))
 
     def copy_to_path(self, path):
-        if not settings.AWS_ENABLED:
-            self._copy_to_path_local(path)
-        else:
-            s3.read_file_to_filesystem(self.bucket_name, self.s3_path, path)
+        s3.read_file_to_filesystem(self.bucket_name, self.s3_path, path)
 
     class Meta(IdeModel.Meta):
         abstract = True

ide/models/scriptfile.py

@@ -10,7 +10,6 @@ class ScriptFile(S3File):
     """ ScriptFiles add support to TextFiles for last-modified timestamps and code folding """
     last_modified = models.DateTimeField(blank=True, null=True, auto_now=True)
     folded_lines = models.TextField(default="[]")
-    _create_local_if_not_exists = True
 
     def was_modified_since(self, expected_modification_time):
         if isinstance(expected_modification_time, int):

ide/tasks/archive.py

@@ -63,16 +63,9 @@ def create_archive(project_id):
 
         send_td_event('cloudpebble_export_project', project=project)
 
-        if not settings.AWS_ENABLED:
-            outfile = '%s%s/%s.zip' % (settings.EXPORT_DIRECTORY, u, prefix)
-            os.makedirs(os.path.dirname(outfile), 0755)
-            shutil.copy(filename, outfile)
-            os.chmod(outfile, 0644)
-            return '%s%s/%s.zip' % (settings.EXPORT_ROOT, u, prefix)
-        else:
-            outfile = '%s/%s.zip' % (u, prefix)
-            s3.upload_file('export', outfile, filename, public=True, content_type='application/zip')
-            return '%s%s' % (settings.EXPORT_ROOT, outfile)
+        outfile = '%s/%s.zip' % (u, prefix)
+        s3.upload_file('export', outfile, filename, public=True, content_type='application/zip')
+        return '%s%s' % (settings.EXPORT_ROOT, outfile)
 
 
 @task(acks_late=True)
@@ -105,7 +98,7 @@ def get_filename_variant(file_name, resource_suffix_map):
     split = file_name_parts[0].split("~")
     tags = split[1:]
     try:
-        ids = [resource_suffix_map['~'+tag] for tag in tags]
+        ids = [resource_suffix_map['~' + tag] for tag in tags]
     except KeyError as key:
         raise ValueError('Unrecognised tag %s' % key)
     root_file_name = split[0] + file_name_parts[1]
@@ -146,7 +139,6 @@ def do_import_archive(project_id, archive, delete_project=False):
                     raise Exception("Too many files in zip file.")
                 file_list = [x.filename for x in contents]
 
-
                 base_dir = find_project_root(file_list)
                 dir_end = len(base_dir)
 
@@ -223,9 +215,9 @@ def make_valid_filename(zip_entry):
                                 filename = make_valid_filename(zipitem)
                                 if filename is False or not filename.startswith(RES_PATH):
                                     continue
-                                filename = filename[len(RES_PATH)+1:]
+                                filename = filename[len(RES_PATH) + 1:]
                                 try:
-                                    extracted = z.open("%s%s/%s"%(base_dir, RES_PATH, filename))
+                                    extracted = z.open("%s%s/%s" % (base_dir, RES_PATH, filename))
                                 except KeyError:
                                     print "Failed to open %s" % filename
                                     continue

utils/s3.py

@@ -4,6 +4,7 @@
 from django.conf import settings
 import urllib
 
+
 def _ensure_bucket_exists(s3, bucket):
     try:
         s3.create_bucket(bucket)
@@ -12,57 +13,43 @@ def _ensure_bucket_exists(s3, bucket):
     else:
         print "Created bucket %s" % bucket
 
-if settings.AWS_ENABLED:
-    if settings.AWS_S3_FAKE_S3 is None:
-        _s3 = boto.connect_s3(settings.AWS_ACCESS_KEY_ID, settings.AWS_SECRET_ACCESS_KEY)
-    else:
-        host, port = (settings.AWS_S3_FAKE_S3.split(':', 2) + [80])[:2]
-        port = int(port)
-        _s3 = boto.connect_s3("key_id", "secret_key", is_secure=False, port=port,
-                              host=host, calling_format=OrdinaryCallingFormat())
-        _ensure_bucket_exists(_s3, settings.AWS_S3_SOURCE_BUCKET)
-        _ensure_bucket_exists(_s3, settings.AWS_S3_EXPORT_BUCKET)
-        _ensure_bucket_exists(_s3, settings.AWS_S3_BUILDS_BUCKET)
-
-    _buckets = {
-        'source': _s3.get_bucket(settings.AWS_S3_SOURCE_BUCKET),
-        'export': _s3.get_bucket(settings.AWS_S3_EXPORT_BUCKET),
-        'builds': _s3.get_bucket(settings.AWS_S3_BUILDS_BUCKET),
-    }
-else:
-    _s3 = None
-    _buckets = None
 
+if settings.AWS_S3_FAKE_S3 is None:
+    _s3 = boto.connect_s3(settings.AWS_ACCESS_KEY_ID, settings.AWS_SECRET_ACCESS_KEY)
+else:
+    host, port = (settings.AWS_S3_FAKE_S3.split(':', 2) + [80])[:2]
+    port = int(port)
+    _s3 = boto.connect_s3("key_id", "secret_key", is_secure=False, port=port,
+                          host=host, calling_format=OrdinaryCallingFormat())
+    _ensure_bucket_exists(_s3, settings.AWS_S3_SOURCE_BUCKET)
+    _ensure_bucket_exists(_s3, settings.AWS_S3_EXPORT_BUCKET)
+    _ensure_bucket_exists(_s3, settings.AWS_S3_BUILDS_BUCKET)
 
-def _requires_aws(fn):
-    if settings.AWS_ENABLED:
-        return fn
-    else:
-        def complain(*args, **kwargs):
-            raise Exception("AWS_ENABLED must be True to call %s" % fn.__name__)
-        return complain
+_buckets = {
+    'source': _s3.get_bucket(settings.AWS_S3_SOURCE_BUCKET),
+    'export': _s3.get_bucket(settings.AWS_S3_EXPORT_BUCKET),
+    'builds': _s3.get_bucket(settings.AWS_S3_BUILDS_BUCKET),
+}
 
 
-@_requires_aws
 def read_file(bucket_name, path):
     bucket = _buckets[bucket_name]
     key = bucket.get_key(path)
     return key.get_contents_as_string()
 
 
-@_requires_aws
 def read_file_to_filesystem(bucket_name, path, destination):
     bucket = _buckets[bucket_name]
     key = bucket.get_key(path)
     key.get_contents_to_filename(destination)
 
-@_requires_aws
+
 def delete_file(bucket_name, path):
     bucket = _buckets[bucket_name]
     key = bucket.get_key(path)
     key.delete()
 
-@_requires_aws
+
 def save_file(bucket_name, path, value, public=False, content_type='application/octet-stream'):
     bucket = _buckets[bucket_name]
     key = Key(bucket)
@@ -76,7 +63,6 @@ def save_file(bucket_name, path, value, public=False, content_type='application/
     key.set_contents_from_string(value, policy=policy, headers={'Content-Type': content_type})
 
 
-@_requires_aws
 def upload_file(bucket_name, dest_path, src_path, public=False, content_type='application/octet-stream', download_filename=None):
     bucket = _buckets[bucket_name]
     key = Key(bucket)
@@ -92,12 +78,11 @@ def upload_file(bucket_name, dest_path, src_path, public=False, content_type='ap
     }
 
     if download_filename is not None:
-        headers['Content-Disposition'] = 'attachment;filename="%s"' % download_filename.replace(' ','_')
+        headers['Content-Disposition'] = 'attachment;filename="%s"' % download_filename.replace(' ', '_')
 
     key.set_contents_from_filename(src_path, policy=policy, headers=headers)
 
 
-@_requires_aws
 def get_signed_url(bucket_name, path, headers=None):
     bucket = _buckets[bucket_name]
     key = bucket.get_key(path)