Merge pull request #247 from /issues/245-246-lowercase-namespace

Fix namespace casing and `inconsistencies_detected` retries

Author: hannes-ucsc

agent/src/cgcloud/agent/cli.py

@@ -32,12 +32,11 @@ def main( ):
                         required=True,
                         help='Optional prefix for naming EC2 resource like instances, images, '
                              'volumes, etc. Use this option to create a separate namespace in '
-                             'order to avoid collisions, e.g. when running tests. The default '
-                             'represents the root namespace. The value of the environment '
-                             'variable CGCLOUD_NAMESPACE, if that variable is present, overrides '
-                             'the default. The string __me__ anywhere in the namespace will be '
-                             'replaced by the name of the IAM user whose credentials are used to '
-                             'issue requests to AWS.' )
+                             'order to avoid collisions, e.g. when running tests. The value of '
+                             'the environment variable CGCLOUD_NAMESPACE, if that variable is '
+                             'present, overrides the default. The string __me__ anywhere in the '
+                             'namespace will be replaced by the name of the IAM user whose '
+                             'credentials are used to issue requests to AWS.' )
     default_zone = os.environ.get( 'CGCLOUD_ZONE', None )
     group.add_argument( '--zone', '-z', metavar='AVAILABILITY_ZONE',
                         default=default_zone,

core/src/cgcloud/core/box.py

@@ -31,7 +31,7 @@
 from paramiko.client import MissingHostKeyPolicy
 
 from cgcloud.core.project import project_artifacts
-from cgcloud.lib import aws_d64
+from cgcloud.lib import aws_d32
 from cgcloud.lib.context import Context
 from cgcloud.lib.ec2 import (ec2_instance_types,
                              wait_instances_running,
@@ -1402,7 +1402,7 @@ def _hash_iam_role_name( self, iam_role_name ):
         # name to the IAM role name. Prependi the prefix here and in _get_iam_ec2_role to be
         # backwards-compatible PassRole statements generated by older versions of CGCloud.
         return '-'.join( [ self.iam_role_name_prefix,
-                           aws_d64.encode( hashlib.sha1( iam_role_name ).digest( )[ 0:8 ] ) ] )
+                           aws_d32.encode( hashlib.sha1( iam_role_name ).digest( )[ 0:8 ] ) ] )
 
     iam_role_name_prefix = 'cgcloud'
 

core/src/cgcloud/core/commands.py

@@ -59,7 +59,11 @@ def __init__( self, application, **kwargs ):
         self.option( '--namespace', '-n', metavar='PREFIX', default=self.default_namespace,
                      help=heredoc( """Optional prefix for naming EC2 resource like instances,
                      images, volumes, etc. Use this option to create a separate namespace in
-                     order to avoid collisions, e.g. when running tests. The value of the
+                     order to avoid collisions, e.g. when running tests. A namespace begins with
+                     a slash, followed by zero or more names, each name followed by a slash. Note
+                     that this implies that the namespace begins and ends with a slash. Each name
+                     must begin with a a digit or lowercase letter followed by zero or more
+                     digits, lowercase letters, periods, underscores or dashes. The value of the
                      environment variable CGCLOUD_NAMESPACE, if that variable is present,
                      overrides the default. The string __me__ anywhere in the namespace will be
                      replaced by the name of the IAM user whose credentials are used to issue

lib/src/cgcloud/lib/__init__.py

@@ -1,5 +1,5 @@
-from bd2k.util.d64 import D64
+from bd2k.util.d32 import standard as d32
 
-aws_d64 = D64( '.-' )  # hopefully the dot is supported for all AWS resource names
+aws_d32 = d32
 
-test_namespace_suffix_length = 11
+test_namespace_suffix_length = 13

lib/src/cgcloud/lib/context.py

@@ -34,7 +34,7 @@ class Context( object ):
     """
     availability_zone_re = re.compile( r'^([a-z]{2}-[a-z]+-[1-9][0-9]*)([a-z])$' )
 
-    name_prefix_re = re.compile( r'^(/([0-9a-zA-Z.-][_0-9a-zA-Z.-]*))*' )
+    name_prefix_re = re.compile( r'^(/([0-9a-z][0-9a-z._-]*))*' )
     name_re = re.compile( name_prefix_re.pattern + '/?$' )
     namespace_re = re.compile( name_prefix_re.pattern + '/$' )
 
@@ -115,7 +115,7 @@ def __init__( self, availability_zone, namespace ):
         ValueError: 'ascii' codec can't encode characters in position 2-3: ordinal not in range(128)
 
         >>> import string
-        >>> component = string.ascii_letters + string.digits + '-_.'
+        >>> component = string.ascii_lowercase + string.digits + '-_.'
         >>> namespace = '/' + component + '/'
         >>> Context('us-west-1b', namespace=namespace).namespace == namespace
         True
@@ -642,6 +642,7 @@ def resolve_me( self, s, drop_hostname=True ):
                     'http://boto.readthedocs.org/en/latest/boto_config_tut.html' )
             if drop_hostname:
                 me = self.drop_hostname( me )
+            me = me.lower() # namespaces must be lower case
             return s.replace( placeholder, me )
         else:
             return s

lib/src/cgcloud/lib/ec2.py

@@ -9,7 +9,7 @@
 from boto.ec2.ec2object import TaggedEC2Object
 from boto.ec2.instance import Instance
 from boto.ec2.spotinstancerequest import SpotInstanceRequest
-from boto.exception import EC2ResponseError
+from boto.exception import EC2ResponseError, BotoServerError
 
 from cgcloud.lib.util import UserError
 
@@ -26,7 +26,7 @@ def not_found( e ):
 
 def retry_ec2( retry_after=a_short_time, retry_for=10 * a_short_time, retry_while=not_found ):
     t = retry_after
-    return retry( delays=(t,t,t*2,t*4), timeout=retry_for, predicate=retry_while )
+    return retry( delays=(t, t, t * 2, t * 4), timeout=retry_for, predicate=retry_while )
 
 
 class EC2VolumeHelper( object ):
@@ -343,11 +343,12 @@ def spot_request_not_found( e ):
 
 
 def create_spot_instances( ec2, price, image_id, spec,
-                           num_instances=1, timeout=None, tentative=False, tags=None):
+                           num_instances=1, timeout=None, tentative=False, tags=None ):
     """
     :rtype: Iterator[list[Instance]]
     """
-    def spotRequestNotFound(e):
+
+    def spotRequestNotFound( e ):
         return e.error_code == "InvalidSpotInstanceRequestID.NotFound"
 
     for attempt in retry_ec2( retry_for=a_long_time,
@@ -357,9 +358,9 @@ def spotRequestNotFound(e):
 
     if tags is not None:
         for requestID in (request.id for request in requests):
-            for attempt in retry_ec2(retry_while=spotRequestNotFound):
+            for attempt in retry_ec2( retry_while=spotRequestNotFound ):
                 with attempt:
-                    ec2.create_tags([requestID], tags)
+                    ec2.create_tags( [ requestID ], tags )
 
     num_active, num_other = 0, 0
     # noinspection PyUnboundLocalVariable,PyTypeChecker
@@ -391,6 +392,7 @@ def spotRequestNotFound(e):
 
 
 def inconsistencies_detected( e ):
+    if not isinstance( e, BotoServerError ): return False
     if e.code == 'InvalidGroup.NotFound': return True
     m = e.error_message.lower( )
     return 'invalid iam instance profile' in m or 'no associated iam roles' in m

lib/src/cgcloud/lib/test/__init__.py

@@ -5,7 +5,7 @@
 
 from boto.utils import get_instance_metadata
 
-from cgcloud.lib import aws_d64, test_namespace_suffix_length
+from cgcloud.lib import aws_d32, test_namespace_suffix_length
 from cgcloud.lib.context import Context
 from cgcloud.lib.ec2 import running_on_ec2
 
@@ -26,11 +26,11 @@ def setUpClass( cls ):
         if running_on_ec2( ):
             os.environ.setdefault( 'CGCLOUD_ZONE',
                                    get_instance_metadata( )[ 'placement' ][ 'availability-zone' ] )
-        # Using the d64 of a binary string that starts with a 4-byte, big-endian time stamp
+        # Using the d32 of a binary string that starts with a 4-byte, big-endian time stamp
         # yields compact names whose lexicographical sorting is consistent with the historical
         # order. We add the process ID so we can run tests concurrently in child processes using
         # the pytest-xdist plugin.
-        suffix = aws_d64.encode( pack( '>II', int( time.time( ) ), os.getpid( ) ) )
+        suffix = aws_d32.encode( pack( '>II', int( time.time( ) ), os.getpid( ) ) )
         assert len( suffix ) == test_namespace_suffix_length
         cls.__namespace = '/test/%s/' % suffix
         os.environ.setdefault( 'CGCLOUD_NAMESPACE', cls.__namespace )

version.py

@@ -1,5 +1,5 @@
 cgcloud_version = '1.6.0a1'
-bd2k_python_lib_dep = 'bd2k-python-lib>=1.14a1.dev35'
+bd2k_python_lib_dep = 'bd2k-python-lib>=1.14a1.dev37'
 boto_dep = 'boto==2.38.0'
 fabric_dep = 'Fabric==1.10.3'
 s3am_dep = 's3am==2.0a1.dev105'