Replace boto with boto3 (#167)

Author: asalajan

requirements.txt

@@ -1,18 +1,19 @@
 simpledi==0.4.1
-awscli==1.29.12
-boto3==1.28.12
-boto==2.49.0
-ansible==8.5.0
+awscli==1.32.6
+boto3==1.34.6
+botocore==1.34.6
+urllib3==2.0.7
+ansible==8.7.0
 azure-common==1.1.28
 azure==4.0.0
 msrestazure==0.6.4
-Jinja2==3.1.2
+Jinja2==3.1.4
 hashmerge
 python-consul
-hvac==1.1.1
+hvac==1.2.1
 passgen
 inflection==0.5.1
 kubernetes==26.1.0
-himl==0.15.0
+himl==0.15.2
 six
 GitPython==3.1.*

src/ops/cli/inventory.py

@@ -53,7 +53,7 @@ def run(self, args, extra_args):
             group_names = [group.name for group in host.get_groups()]
             group_names = sorted(group_names)
             group_string = ", ".join(group_names)
-            host_id = host.vars.get('ec2_id', '')
+            host_id = host.vars.get('ec2_InstanceId', '')
             if host_id != '':
                 name_and_id = "%s -- %s" % (stringc(host.name,
                                                     'blue'), stringc(host_id, 'blue'))

src/ops/inventory/ec2inventory.py

@@ -11,26 +11,24 @@
 import json
 import re
 import sys
-import os
 
-import boto
-from boto import ec2
-from boto.pyami.config import Config
-
-from six import iteritems, string_types, integer_types
+import boto3
+from botocore.exceptions import NoRegionError, NoCredentialsError, PartialCredentialsError
 
 
 class Ec2Inventory(object):
-    def _empty_inventory(self):
+    @staticmethod
+    def _empty_inventory():
         return {"_meta": {"hostvars": {}}}
 
-    def __init__(self, boto_profile, regions, filters={}, bastion_filters={}):
+    def __init__(self, boto_profile, regions, filters=None, bastion_filters=None):
 
-        self.filters = filters
+        self.filters = filters or []
         self.regions = regions.split(',')
         self.boto_profile = boto_profile
-        self.bastion_filters = bastion_filters
+        self.bastion_filters = bastion_filters or []
         self.group_callbacks = []
+        self.boto3_session = self.create_boto3_session(boto_profile)
 
         # Inventory grouped by instance IDs, tags, security groups, regions,
         # and availability zones
@@ -39,6 +37,25 @@ def __init__(self, boto_profile, regions, filters={}, bastion_filters={}):
         # Index of hostname (address) to instance ID
         self.index = {}
 
+    def create_boto3_session(self, profile_name):
+        try:
+            # Use the profile to create a session
+            session = boto3.Session(profile_name=profile_name)
+
+            # Verify region
+            if not self.regions:
+                if not session.region_name:
+                    raise NoRegionError
+                self.regions = [session.region_name]
+
+        except NoRegionError:
+            sys.exit(f"Region not specified and could not be determined for profile: {profile_name}")
+        except (NoCredentialsError, PartialCredentialsError):
+            sys.exit(f"Credentials not found or incomplete for profile: {profile_name}")
+        except Exception as e:
+            sys.exit(f"An error occurred: {str(e)}")
+        return session
+
     def get_as_json(self):
         self.do_api_calls_update_cache()
         return self.json_format_dict(self.inventory, True)
@@ -55,20 +72,20 @@ def exclude(self, *args):
     def group(self, *args):
         self.group_callbacks.extend(args)
 
-    def find_bastion_box(self, conn):
+    def find_bastion_box(self, ec2_client):
         """
         Find ips for the bastion box
         """
 
-        if not self.bastion_filters.values():
+        if not self.bastion_filters:
             return
 
-        self.bastion_filters['instance-state-name'] = 'running'
+        self.bastion_filters.append({'Name': 'instance-state-name', 'Values': ['running']})
 
-        for reservation in conn.get_all_instances(
-                filters=self.bastion_filters):
-            for instance in reservation.instances:
-                return instance.ip_address
+        reservations = ec2_client.describe_instances(Filters=self.bastion_filters)['Reservations']
+        for reservation in reservations:
+            for instance in reservation['Instances']:
+                return instance['PublicIpAddress']
 
     def do_api_calls_update_cache(self):
         """ Do API calls to each region, and save data in cache files """
@@ -80,92 +97,67 @@ def get_instances_by_region(self, region):
         """Makes an AWS EC2 API call to the list of instances in a particular
         region
         """
+        ec2_client = self.boto3_session.client('ec2', region_name=region)
 
-        try:
-            cfg = Config()
-            cfg.load_credential_file(os.path.expanduser("~/.aws/credentials"))
-            cfg.load_credential_file(os.path.expanduser("~/.aws/config"))
-            session_token = cfg.get(self.boto_profile, "aws_session_token")
-
-            conn = ec2.connect_to_region(
-                region,
-                security_token=session_token,
-                profile_name=self.boto_profile)
-
-            # connect_to_region will fail "silently" by returning None if the
-            # region name is wrong or not supported
-            if conn is None:
-                sys.exit(
-                    "region name: {} likely not supported, or AWS is down. "
-                    "connection to region failed.".format(region))
+        reservations = ec2_client.describe_instances(Filters=self.filters)['Reservations']
 
-            reservations = conn.get_all_instances(filters=self.filters)
-
-            bastion_ip = self.find_bastion_box(conn)
-
-            instances = []
-            for reservation in reservations:
-                instances.extend(reservation.instances)
-
-            # sort the instance based on name and index, in this order
-            def sort_key(instance):
-                name = instance.tags.get('Name', '')
-                return "{}-{}".format(name, instance.id)
-
-            for instance in sorted(instances, key=sort_key):
-                self.add_instance(bastion_ip, instance, region)
+        bastion_ip = self.find_bastion_box(ec2_client)
+        instances = []
+        for reservation in reservations:
+            instances.extend(reservation['Instances'])
 
-        except boto.provider.ProfileNotFoundError as e:
-            raise Exception(
-                "{}, configure it with 'aws configure --profile {}'".format(e.message, self.boto_profile))
+        # sort the instance based on name and index, in this order
+        def sort_key(instance):
+            name = next((tag['Value'] for tag in instance.get('Tags', [])
+                         if tag['Key'] == 'Name'), '')
+            return "{}-{}".format(name, instance['InstanceId'])
 
-        except boto.exception.BotoServerError as e:
-            sys.exit(e)
+        for instance in sorted(instances, key=sort_key):
+            self.add_instance(bastion_ip, instance, region)
 
     def get_instance(self, region, instance_id):
         """ Gets details about a specific instance """
-        conn = ec2.connect_to_region(region)
-
+        ec2_client = self.boto3_session.client('ec2', region_name=region)
         # connect_to_region will fail "silently" by returning None if the
         # region name is wrong or not supported
-        if conn is None:
+        if ec2_client is None:
             sys.exit(
                 "region name: %s likely not supported, or AWS is down. "
                 "connection to region failed." % region
             )
 
-        reservations = conn.get_all_instances([instance_id])
+        reservations = ec2_client.describe_instances(InstanceIds=[instance_id])['Reservations']
         for reservation in reservations:
-            for instance in reservation.instances:
+            for instance in reservation['Instances']:
                 return instance
 
     def add_instance(self, bastion_ip, instance, region):
         """
-        :type instance: boto.ec2.instance.Instance
+        :type instance: dict
         """
 
         # Only want running instances unless all_instances is True
-        if instance.state != 'running':
+        if instance['State']['Name'] != 'running':
             return
 
         # Use the instance name instead of the public ip
-        dest = instance.tags.get('Name', instance.ip_address)
+        dest = next((tag['Value'] for tag in instance.get('Tags', []) if tag['Key'] == 'Name'), instance.get('PublicIpAddress'))
         if not dest:
             return
 
-        if bastion_ip and bastion_ip != instance.ip_address:
-            ansible_ssh_host = bastion_ip + "--" + instance.private_ip_address
-        elif instance.ip_address:
-            ansible_ssh_host = instance.ip_address
+        if bastion_ip and bastion_ip != instance.get('PublicIpAddress'):
+            ansible_ssh_host = bastion_ip + "--" + instance.get('PrivateIpAddress')
+        elif instance.get('PublicIpAddress'):
+            ansible_ssh_host = instance.get('PublicIpAddress')
         else:
-            ansible_ssh_host = instance.private_ip_address
+            ansible_ssh_host = instance.get('PrivateIpAddress')
 
         # Add to index and append the instance id afterwards if it's already
         # there
         if dest in self.index:
-            dest = dest + "-" + instance.id.replace("i-", "")
+            dest = dest + "-" + instance['InstanceId'].replace("i-", "")
 
-        self.index[dest] = [region, instance.id]
+        self.index[dest] = [region, instance['InstanceId']]
 
         # group with dynamic groups
         for grouping in set(self.group_callbacks):
@@ -175,9 +167,9 @@ def add_instance(self, bastion_ip, instance, region):
                     self.push(self.inventory, group, dest)
 
         # Group by all tags
-        for tag in instance.tags.values():
-            if tag:
-                self.push(self.inventory, tag, dest)
+        for tag in instance.get('Tags', []):
+            if tag['Value']:
+                self.push(self.inventory, tag['Value'], dest)
 
         # Inventory: Group by region
         self.push(self.inventory, region, dest)
@@ -186,56 +178,39 @@ def add_instance(self, bastion_ip, instance, region):
         self.push(self.inventory, ansible_ssh_host, dest)
 
         # Inventory: Group by availability zone
-        self.push(self.inventory, instance.placement, dest)
+        self.push(self.inventory, instance['Placement']['AvailabilityZone'], dest)
 
-        self.inventory["_meta"]["hostvars"][dest] = self.get_host_info_dict_from_instance(
-            instance)
+        self.inventory["_meta"]["hostvars"][dest] = self.get_host_info_dict_from_instance(instance)
         self.inventory["_meta"]["hostvars"][dest]['ansible_ssh_host'] = ansible_ssh_host
 
     def get_host_info_dict_from_instance(self, instance):
         instance_vars = {}
-        for key in vars(instance):
-            value = getattr(instance, key)
-            key = self.to_safe('ec2_' + key)
-
-            # Handle complex types
-            # state/previous_state changed to properties in boto in
-            # https://github.com/boto/boto/commit/a23c379837f698212252720d2af8dec0325c9518
-            if key == 'ec2__state':
-                instance_vars['ec2_state'] = instance.state or ''
-                instance_vars['ec2_state_code'] = instance.state_code
-            elif key == 'ec2__previous_state':
-                instance_vars['ec2_previous_state'] = instance.previous_state or ''
-                instance_vars['ec2_previous_state_code'] = instance.previous_state_code
-            elif type(value) in integer_types or isinstance(value, bool):
-                instance_vars[key] = value
-            elif type(value) in string_types:
-                instance_vars[key] = value.strip()
+        for key, value in instance.items():
+            safe_key = self.to_safe('ec2_' + key)
+
+            if key == 'State':
+                instance_vars['ec2_state'] = value['Name']
+                instance_vars['ec2_state_code'] = value['Code']
+            elif isinstance(value, (int, bool)):
+                instance_vars[safe_key] = value
+            elif isinstance(value, str):
+                instance_vars[safe_key] = value.strip()
             elif value is None:
-                instance_vars[key] = ''
-            elif key == 'ec2_region':
-                instance_vars[key] = value.name
-            elif key == 'ec2__placement':
-                instance_vars['ec2_placement'] = value.zone
-            elif key == 'ec2_tags':
-                for k, v in iteritems(value):
-                    key = self.to_safe('ec2_tag_' + k)
-                    instance_vars[key] = v
-            elif key == 'ec2_groups':
-                group_ids = []
-                group_names = []
-                for group in value:
-                    group_ids.append(group.id)
-                    group_names.append(group.name)
+                instance_vars[safe_key] = ''
+            elif key == 'Placement':
+                instance_vars['ec2_placement'] = value['AvailabilityZone']
+            elif key == 'Tags':
+                for tag in value:
+                    tag_key = self.to_safe('ec2_tag_' + tag['Key'])
+                    instance_vars[tag_key] = tag['Value']
+            elif key == 'SecurityGroups':
+                group_ids = [group['GroupId'] for group in value]
+                group_names = [group['GroupName'] for group in value]
                 instance_vars["ec2_security_group_ids"] = ','.join(group_ids)
-                instance_vars["ec2_security_group_names"] = ','.join(
-                    group_names)
-        # add non ec2 prefix private ip address that are being used in cross provider command
-        # e.g ssh, sync
-        instance_vars['private_ip'] = instance_vars.get(
-            'ec2_private_ip_address', '')
-        instance_vars['private_ip_address'] = instance_vars.get(
-            'ec2_private_ip_address', '')
+                instance_vars["ec2_security_group_names"] = ','.join(group_names)
+
+        instance_vars['private_ip'] = instance.get('PrivateIpAddress', '')
+        instance_vars['private_ip_address'] = instance.get('PrivateIpAddress', '')
         return instance_vars
 
     def get_host_info(self):

src/ops/inventory/plugin/cns.py

@@ -27,13 +27,13 @@ def cns(args):
                 region=region,
                 boto_profile=profile,
                 cache=args.get('cache', 3600 * 24),
-                filters={
-                    'tag:cluster': cns_cluster
-                },
-                bastion={
-                    'tag:cluster': cns_cluster,
-                    'tag:role': 'bastion'
-                }
+                filters=[
+                    {'Name': 'tag:cluster', 'Values': [cns_cluster]}
+                ],
+                bastion=[
+                    {'Name': 'tag:cluster', 'Values': [cns_cluster]},
+                    {'Name': 'tag:role', 'Values': ['bastion']}
+                ]
             ))
 
             merge_inventories(result, json.loads(jsn))

src/ops/inventory/plugin/ec2.py

@@ -12,15 +12,17 @@
 
 
 def ec2(args):
-    filters = args.get('filters', {})
-    bastion_filters = args.get('bastion', {})
+    filters = args.get('filters', [])
+    bastion_filters = args.get('bastion', [])
 
     if args.get('cluster') and not args.get('filters'):
-        filters['tag:cluster'] = args.get('cluster')
+        filters = [{'Name': 'tag:cluster', 'Values': [args.get('cluster')]}]
 
     if args.get('cluster') and not args.get('bastion'):
-        bastion_filters['tag:cluster'] = args.get('cluster')
-        bastion_filters['tag:role'] = 'bastion'
+        bastion_filters = [
+            {'Name': 'tag:cluster', 'Values': [args.get('cluster')]},
+            {'Name': 'tag:role', 'Values': ['bastion']}
+        ]
 
     return Ec2Inventory(boto_profile=args['boto_profile'],
                         regions=args['region'],