major update

Author: awsandy

aws2tf.py

@@ -3,10 +3,14 @@
 import concurrent.futures
 import json
 import datetime
+import logging
+from tqdm import tqdm
+
+log = logging.getLogger('aws2tf')
 
 
 def build_lists():
-    print("Building core resource lists ...")
+    log.info("Building core resource lists ...")
     context.tracking_message="Stage 2 of 10, Building core resource lists ..."
 
 
@@ -20,7 +24,7 @@ def fetch_lambda_data():
                 response.extend(page['Functions'])
             return [('lambda', j['FunctionName']) for j in response]
         except Exception as e:
-            print("Error fetching Lambda data:", e)
+            log.error("Error fetching Lambda data: %s %s",  e)
             return []
 
 
@@ -34,7 +38,7 @@ def fetch_vpc_data():
             context.vpcs=response
             return [('vpc', j['VpcId']) for j in response]
         except Exception as e:
-            print("Error fetching ec2 data:", e)
+            log.error("Error fetching ec2 data: %s %s",  e)
             return []
 
     def fetch_s3_data():
@@ -46,7 +50,7 @@ def fetch_s3_data():
                 response.extend(page['Buckets'])
             return [('s3', j['Name']) for j in response]
         except Exception as e:
-            print("Error fetching s3 data:", e)
+            log.error("Error fetching s3 data: %s %s",  e)
             return []
 
     def fetch_sg_data():
@@ -58,7 +62,7 @@ def fetch_sg_data():
                 response.extend(page['SecurityGroups'])
             return [('sg', j['GroupId']) for j in response]
         except Exception as e:
-            print("Error fetching SG data:", e)
+            log.error("Error fetching SG data: %s %s",  e)
             return []
 
 
@@ -75,7 +79,7 @@ def fetch_subnet_data():
                json.dump(response, f, indent=2, default=str)
             return [('subnet', j['SubnetId']) for j in response]
         except Exception as e:
-            print("Error fetching vpc data:", e)
+            log.error("Error fetching vpc data: %s %s",  e)
             return []
 
     def fetch_tgw_data():
@@ -87,7 +91,7 @@ def fetch_tgw_data():
                 response.extend(page['TransitGateways'])
             return [('tgw', j['TransitGatewayId']) for j in response]
         except Exception as e:
-            print("Error fetching transit gateways:", e)
+            log.error("Error fetching transit gateways: %s %s",  e)
             return []
 
     def fetch_roles_data():
@@ -102,7 +106,7 @@ def fetch_roles_data():
                json.dump(response, f, indent=2, default=str)
             return [('iam', j['RoleName']) for j in response]
         except Exception as e:
-            print("Error fetching vpc data:", e)
+            log.error("Error fetching vpc data: %s %s",  e)
             return []
 
     def fetch_policies_data():
@@ -114,7 +118,19 @@ def fetch_policies_data():
                 response.extend(page['Policies'])
             return [('pol', j['Arn']) for j in response]
         except Exception as e:
-            print("Error fetching vpc data:", e)
+            log.error("Error fetching vpc data: %s %s",  e)
+            return []
+
+    def fetch_instprof_data():
+        try:
+            client = boto3.client('iam',region_name='us-east-1')
+            response = []
+            paginator = client.get_paginator('list_instance_profiles')
+            for page in paginator.paginate():
+                response.extend(page['InstanceProfiles'])
+            return [('inp', j['InstanceProfileName']) for j in response]
+        except Exception as e:
+            log.error("Error fetching vpc data: %s %s",  e)
             return []
 
 
@@ -128,7 +144,8 @@ def fetch_policies_data():
             executor.submit(fetch_subnet_data),
             executor.submit(fetch_tgw_data),
             executor.submit(fetch_roles_data),
-            executor.submit(fetch_policies_data)
+            executor.submit(fetch_policies_data),
+            executor.submit(fetch_instprof_data)
         ]
 
         # Process results as they complete
@@ -149,13 +166,11 @@ def fetch_policies_data():
                     elif resource_type == 's3':
                         client = boto3.client('s3')
                         for _, bucket in result:
-                            #here ? 
-                            #print("Buck from result=",bucket)   
                             try:
                                 ####### problematic call
                                 objs = client.list_objects_v2(Bucket=bucket,MaxKeys=1)      
                             except Exception as e:
-                                print(f"Error details: {e}")
+                                log.error(f"Error details: {e}")
                                 continue
 
                             context.s3list[bucket] = True
@@ -174,6 +189,9 @@ def fetch_policies_data():
                     elif resource_type == 'pol':
                         for _, policy_arn in result:
                             context.policylist[policy_arn] = True
+                    elif resource_type == 'inp':
+                        for _, inst_prof in result:
+                            context.inplist[inst_prof] = True
                 else:
                     # Handle roles data
                     with open('imported/roles.json', 'w') as f:
@@ -195,7 +213,7 @@ def fetch_policies_data():
 def build_secondary_lists(id=None):
     if id is None:
         st1 = datetime.datetime.now()
-        print("Building secondary IAM resource lists ...")
+        log.info("Building secondary IAM resource lists ...")
         context.esttime = (len(context.rolelist) * 3) / 4
         context.tracking_message = "Stage 2 of 10, Building secondary IAM resource lists ..."
 
@@ -214,7 +232,7 @@ def fetch_role_policies(role_name):
                     'inline_policies': inline_policies['PolicyNames'] if inline_policies['PolicyNames'] else False
                 }
             except Exception as e:
-                print(f"Error fetching policies for role {role_name}: {e}")
+                log.error(f"Error fetching policies for role {role_name}: {e}")
                 return {
                     'role_name': role_name,
                     'attached_policies': False,
@@ -223,28 +241,29 @@ def fetch_role_policies(role_name):
 
         # Use ThreadPoolExecutor to parallelize API calls
         rcl = len(context.rolelist)
+        log.info(f"Fetching policies for {rcl} IAM roles...")
+        
         with concurrent.futures.ThreadPoolExecutor(max_workers=context.cores) as executor:
             # Submit all role policy fetch tasks
             future_to_role = {
                 executor.submit(fetch_role_policies, role_name): role_name 
                 for role_name in context.rolelist.keys()
             }
 
-            # Process results as they complete
-            completed = 0
-            for future in concurrent.futures.as_completed(future_to_role):
-                completed += 1
-                context.tracking_message = f"Stage 2 of 10, Building secondary IAM resource lists... {completed} of {rcl}"
-                
+            # Process results with progress bar
+            for future in tqdm(concurrent.futures.as_completed(future_to_role),
+                              total=len(future_to_role),
+                              desc="Fetching IAM policies",
+                              unit="role"):
                 try:
                     result = future.result()
                     role_name = result['role_name']
                     context.attached_role_policies_list[role_name] = result['attached_policies']
                     context.role_policies_list[role_name] = result['inline_policies']
                 except Exception as e:
-                    print(f"Error processing result: {e}")
+                    log.error(f"Error processing result: {e}")
 
         st2 = datetime.datetime.now()
-        print("secondary lists built in " + str(st2 - st1))
+        log.info("secondary lists built in " + str(st2 - st1))
 
     return