feat: Update inventory scripts with new script "all_my_ram_shares" (#103)

* Updated with enhancements due to case-sensitivity in the API

* Updated to remove region being separate from the ocredentials object. Now it's a field within the object.

* Updated exception logic to find elastic load balancers with no security groups attached

* Brought files in line with both Inventory_Script repos

* Updated with changes from customer work...

* Updated all_my_enis.py with some additional cosmetic updates.

* Updated with some additional cosmetic updates.

* Updated all_my_functions.py with proper error type.

* Updates on progress bars, and using the right variables.

* Removed inventory-scripts directory

* Squashed 'inventory-scripts/' content from commit 14d5610

git-subtree-dir: inventory-scripts
git-subtree-split: 14d5610f6414518210d059b1c25da25a5975e4d4

* Changed inventory-scripts to a subtree and updated the gitignore file

* Updated gitignore file

* Removed unneeded files from subtree

* Removed unneeded files

* Updated gitignore again

* Squashed 'inventory-scripts/' changes from 14d5610..eaf0432

eaf0432 Updated RDS script
c0435ab Updated gitignore file
c1d2b36 Updated to only check for enabled policy types. Should make this significantly faster.
0f2838d Merge remote-tracking branch 'origin/combined' into combined
f3cb5a0 Updated small changes - mostly to version numbers
edec2c6 Edit Discovery.md
3aea3d6 Updated files to sync public and private repos

git-subtree-dir: inventory-scripts
git-subtree-split: eaf04325b08f3241feba24183a0459dfde455106

* Squashed 'inventory-scripts/' changes from eaf04325..15d25f3d

15d25f3d Renamed the profile when we can't find one, from "-default-" to "None"
e745d92b Updated phzs script to remove regionality
6990f1d4 Updated orgs script for version
23434747 Updated orgs script to be able to run without a profile, using environment variables
d7637b57 New files

git-subtree-dir: inventory-scripts
git-subtree-split: 15d25f3dfd6e51d4eba18ae5bf5d4f7080f27d51

* Squashed 'inventory-scripts/' changes from 15d25f3d..3e4d66be

3e4d66be Updated the account_class.py library to better support the use of environment variables for credentials, instead of profiles

git-subtree-dir: inventory-scripts
git-subtree-split: 3e4d66be90d9b863b78c7e11e070e2fa3ade5ebb

* Squashed 'inventory-scripts/' changes from 3e4d66be..a2de0ab3

a2de0ab3 Updated tgw script to accommodate env vars as credentials, instead of a profile

git-subtree-dir: inventory-scripts
git-subtree-split: a2de0ab32c28e228363a5ec6a304385e68ad4d0f

* Squashed 'inventory-scripts/' changes from a2de0ab3..d7173c46

d7173c46 Enabled the output to include the connected TGW for all resources.
39802ee8 Updating .gitignore with ignored suffixes
51c18421 Fixed imports
f74a44e0 Updated with encoding for opened files
4d6ba49c Merge branch 'feature/find_global_accelerators' into development Added the .gitallowed file
09869a7b Merge branch 'test/development' into development Adding more test cases from a test/dev branch
d9978000 Added gitallowed file for example secrets (account numbers)
bb20c5c6 Updated the version of the account_class.py library
62a2ecca Updated the account_class.py library to better support the use of environment variables for credentials, instead of profiles
e615c538 Updated gitignore file
3f78e6d7 More test cases

git-subtree-dir: inventory-scripts
git-subtree-split: d7173c4665b2d654a86a46c9c2e76ff2c74df12b

* Squashed 'inventory-scripts/' changes from d7173c46..14e06dd7

14e06dd7 Updated for fqdn typo
55f1ce19 Version updates
a16974ee Added / Updated policy listing
56059efa Added ability to resolve FQDNs to the ENI script
ad7e718c Fixed error when short-form was used
9a477c58 Merge branch 'refs/heads/fix/mod_my_cfnstacksets' into development
cfcd860d Added pytest to the requirements.txt
80d1ab30 Fixed problem for when profile is embedded in the Environment Variables
393c96c0 Made significant changes to all_my_orgs, and associated account classes, and supporting libraries
7ae901e7 Small comment changes
294132f9 Added new script (created by Q) for network visibility...
08be53f6 Updated
683a5276 Updated all_my_functions.py to fix errors when no functions were found
83fe2304 Updated for the case where the profile specified doesn't exist or work properly
95620a8d Fixed the tgw diagram naming issue

git-subtree-dir: inventory-scripts
git-subtree-split: 14e06dd70eebc9edde45aeeff24e305a08cd0df4

Author: paulbayer

inventory-scripts/.gitignore

@@ -352,3 +352,7 @@ src/.package
 
 my_venv/
 /stackoverflow_test/
+.gitallowed
+aws_organization
+*.txt
+*.csv

inventory-scripts/DrawOrg.py

@@ -8,7 +8,7 @@
 from colorama import init, Fore
 from ArgumentsClass import CommonArguments
 
-__version__ = '2025.04.04'
+__version__ = '2025.09.26'
 
 init()
 
@@ -21,7 +21,8 @@
 ou_fillcolor = 'burlywood'
 ou_shape = 'box'
 # aws_policy_type_list = ['SERVICE_CONTROL_POLICY', 'TAG_POLICY', 'BACKUP_POLICY',
-#                         'AISERVICES_OPT_OUT_POLICY', 'CHATBOT_POLICY', 'RESOURCE_CONTROL_POLICY', 'DECLARATIVE_POLICY_EC2']
+#                         'AISERVICES_OPT_OUT_POLICY', 'CHATBOT_POLICY', 'RESOURCE_CONTROL_POLICY', 
+# 							'DECLARATIVE_POLICY_EC2', 'SECURITYHUB_POLICY']
 
 
 #####################
@@ -182,6 +183,8 @@ def create_policy_nodes(f_enabled_policy_types: list):
 				policy_type = 'chatbot'
 			elif policy['Type'] == 'DECLARATIVE_POLICY_EC2':
 				policy_type = 'dcp'
+			elif policy['Type'] == 'SECURITYHUB_POLICY':
+				policy_type = 'sec'
 			else:
 				policy_type = policy['Type']
 
@@ -298,7 +301,6 @@ def traverse_ous_and_accounts(ou_id: str):
 	max_accounts_per_ou = find_max_accounts_per_ou(froot, max_accounts_per_ou)
 
 	# This tries to verticalize the diagram, so it doesn't look like a wide mess
-	dot.attr(rankdir='LR')
 	dot_unflat = dot.unflatten(stagger=round_up(max_accounts_per_ou / 5))
 
 	# Save the diagram to a PNG file

inventory-scripts/Inventory_Modules.py

@@ -4579,7 +4579,9 @@ def display_results(results_list, fdisplay_dict: dict, defaultAction=None, file_
 
 		Enhancements:
 			- How to create a break between rows, like after every account, or Management Org, or region, or whatever...
-			- How to do sub-sections, where there is more data to show per row...  
+
+		TODO: Documentation on how to do the sub-sections - which has been written, but not explained. 
+		- How to do sub-sections, where there is more data to show per row...  
 	"""
 
 	def handle_list():
@@ -4912,6 +4914,7 @@ def get_all_credentials(fProfiles: list = None, fTiming: bool = False, fSkipProf
 	import logging
 	from account_class import aws_acct_access
 	# from time import time
+	from botocore.exceptions import NoCredentialsError
 	from colorama import init, Fore
 
 	init()
@@ -4930,11 +4933,15 @@ def get_all_credentials(fProfiles: list = None, fTiming: bool = False, fSkipProf
 		fRegionList = ['us-east-1']
 	if fProfiles is None:  # Default use case from the classes
 		print("Getting Accounts to check: ", end='')
-		aws_acct = aws_acct_access()
-		# This doesn't mean the profile "default", this is just what the label for the Org Name will be, since there's no other text
-		profile = '-default-'
+		try:
+			aws_acct = aws_acct_access()
+		except NoCredentialsError as my_Error:
+			logging.error(f"Credentials error: {my_Error}")
+			raise Exception(f"Credential Error")
+		# This doesn't mean the profile "default" or 'None', this is just what the label for the Org Name will be, since there's no other text
+		profile = 'None'
 		RegionList = get_regions3(aws_acct, fRegionList)
-		logging.info(f"No profile string passed in. Using string '-default-'")
+		logging.info(f"No profile string passed in. Using string 'None'")
 		# This should populate the list "AllCreds" with the credentials for the relevant accounts.
 		AllCredentials.extend(get_credentials_for_accounts_in_org(aws_acct, fSkipAccounts, fRootOnly, fAccounts, profile, RegionList, RoleList, fTiming))
 	else:
@@ -5115,7 +5122,7 @@ def run(self):
 	return AllCreds
 
 
-def get_org_accounts_from_profiles(fProfileList):
+def get_org_accounts_from_profiles(fProfileList=None):
 	"""
 	Note that this function returns account_class objects based on the list of profiles passed to it
 	This function is fairly slow since it needs to call the aws_acct_access function for each profile.
@@ -5139,7 +5146,8 @@ def run(self):
 			while True:
 				# Get the work from the queue and expand the tuple
 				profile = self.queue.get()
-				pbar.update()
+				if profile is None:
+					break
 				Account = {'ErrorFlag'   : False,
 				           'Success'     : False,
 				           'RootAcct'    : False,
@@ -5213,27 +5221,49 @@ def run(self):
 						logging.error("Credentials Error")
 						logging.error(my_Error)
 				finally:
+					# logging.info(f"Account: {Account}") ## Remove
+					# print(f"Account: {Account}")  ## Remove
 					self.queue.task_done()
+					pbar.update()
 				AllAccounts.append(Account)
 
+	# print(f"ProfileList from within the mt function: {fProfileList}")  ## Remove
 	AllAccounts = []
 	profilequeue = Queue()
-	# WorkerThreads = len(fProfileList)
-	WorkerThreads = 2
+	if fProfileList:
+		WorkerThreads = min(len(fProfileList), 2)
+	else:
+		WorkerThreads = 1
+	threads = []
 
 	# Create x worker threads
 	for x in range(WorkerThreads):
 		worker = AssembleCredentials(profilequeue)
 		# Setting daemon to True will let the main thread exit even though the workers are blocking
 		worker.daemon = True
 		worker.start()
+		threads.append(worker)
 
-	pbar = tqdm(desc=f'Getting accounts from {len(fProfileList)} profiles',
-	            total=len(fProfileList)
-	            )
-
-	for profile_item in fProfileList:
-		logging.info(f"Queuing profile {profile_item} / {len(fProfileList)} profiles")
-		profilequeue.put(profile_item)
+	if fProfileList is None:
+		logging.info("No profiles were found, using environment variables")
+		logging.error("No profiles were found, using environment variables")
+		pbar = tqdm(desc=f'Getting account from EnvVar', total=1)
+		profilequeue.put('EnvVar')
+	else:
+		logging.info(f"List of profiles being looked at is: {fProfileList}")
+		logging.error(f"List of profiles being looked at is: {fProfileList}")
+		pbar = tqdm(desc=f'Getting accounts from {len(fProfileList)} profiles',
+		            total=len(fProfileList))
+		for profile_item in fProfileList:
+			profilequeue.put(profile_item)
 	profilequeue.join()
+	# Sends 'sentinel value' to threads to tell them we're done
+	for _ in threads:
+		logging.debug(f"Sending 'None' to queue, to ensure all threads finish")
+		profilequeue.put(None)
+	# Waits for all threads to be finished
+	for t in threads:
+		logging.debug(f"Waiting for thread {t.name} to finish")
+		t.join()
+	# print(f"AllAccounts - from within the multi-threaded function, just before return: {AllAccounts}")  ## Remove
 	return AllAccounts

inventory-scripts/RunOnMultiAccounts.py

@@ -151,6 +151,17 @@ def participant_user(faws_acct, create=None, username=None):
 			return_response['ErrorMessage'] = ErrorMessage
 	return return_response
 
+
+def display_firewall_manager(ocredentials):
+	"""
+	Description: Determine if firewall manager is running in this account
+	@param ocredentials: Credentials
+	@return:
+	"""
+	child_acct = aws_acct_access(ocredentials=ocredentials)
+	fw_mgr_client = child_acct.session.client('fms')
+	# response = fw_mgr_client.
+	# response =
 #####################
 
 
@@ -173,6 +184,7 @@ def participant_user(faws_acct, create=None, username=None):
 		Put more commands here... Or you can write functions that represent your commands and call them from here.
 		"""
 		credentials = Inventory_Modules.get_child_access3(aws_acct, account_num, 'us-east-1', ['reinvent-Admin'])
+		display_firewall_manager(credentials)
 		tgt_aws_access = aws_acct_access(ocredentials=credentials)
 		username = 'Paul'
 		user_response = participant_user(tgt_aws_access, username=username)

inventory-scripts/SC_Products_to_CFN_Stacks.py

@@ -92,7 +92,9 @@ def run(self):
 						)
 						# The above command fails if the stack found (by the find_stacks3 function) has been deleted
 						# The following section determines the NEW Account's AccountEmail and AccountID
-						AccountEmail = AccountID = AccountStatus = None
+						AccountEmail = None
+						AccountID = None
+						AccountStatus = None
 						if 'Parameters' in stack_info['Stacks'][0].keys() and len(stack_info['Stacks'][0]['Parameters']) > 0:
 							for y in range(len(stack_info['Stacks'][0]['Parameters'])):
 								if stack_info['Stacks'][0]['Parameters'][y]['ParameterKey'] == 'AccountEmail':
@@ -273,7 +275,7 @@ def main():
 		# 		print(f"{ERASE_LINE}Found {len(result['ProductViewDetails'])} products | Total found: {len(prod_ids)}", end='\r')
 		# print()
 		for product in prod_ids:
-			if product['ProductViewSummary']['Name'].find('Account-Vending-Machine') > 0:
+			if product['ProductViewSummary']['Name'].find('Account-Vending-Machine') > 0 or product['ProductViewSummary']['Name'].find('AWS Control Tower Account Factory') > 0:
 				AVM_prod_id = product['ProductViewSummary']['ProductId']
 	elif pFragment is not None and not pExact:
 		result = client_sc.search_products_as_admin()

inventory-scripts/Tests/common_test_functions.py

@@ -1,6 +1,6 @@
 from botocore import client
 from botocore import session
-import pytest
+# import pytest
 
 ERASE_LINE = '\x1b[2K'
 
@@ -28,7 +28,10 @@ def AWSKeyID_from_AWSAccount(AWSAccountNumber):
 	return AWSKey
 
 
-def _amend_create_boto3_session(test_data, mocker):
+def _amend_create_boto3_session(test_data, verbosity, mocker):
+	import logging
+	logging.basicConfig(level=verbosity, format="[%(filename)s:%(lineno)s - %(funcName)20s() ] %(message)s")
+
 	orig = session.Session.create_client
 
 	def amend_create_client(
@@ -46,12 +49,12 @@ def amend_create_client(
 			):
 		# Intercept boto3 Session, in hopes of sending back a client that includes the Account Number
 		# if aws_access_key_id == '*****AccessKeyHere*****':
-		print(test_data['FunctionName'])
+		logging.info(test_data['FunctionName'])
 		if aws_access_key_id == 'MeantToFail':
-			print(f"Failed Access Key: {aws_access_key_id}")
+			logging.info(f"Failed Access Key: {aws_access_key_id}")
 			return ()
 		else:
-			print(f"Not Failed Access Key: {aws_access_key_id}")
+			logging.info(f"Not Failed Access Key: {aws_access_key_id}")
 			return_response = orig(self,
 			                       service_name,
 			                       region_name,
@@ -69,7 +72,10 @@ def amend_create_client(
 	print()
 
 
-def _amend_make_api_call_orig(test_key, test_value, mocker):
+def _amend_make_api_call_orig(test_key, test_value, verbosity, mocker):
+	import logging
+	logging.basicConfig(level=verbosity, format="[%(filename)s:%(lineno)s - %(funcName)20s() ] %(message)s")
+
 	orig = client.BaseClient._make_api_call
 
 	def amend_make_api_call(self, operation_name, kwargs):
@@ -81,23 +87,26 @@ def amend_make_api_call(self, operation_name, kwargs):
 			if isinstance(test_value, Exception):
 				raise test_value
 			# Implied break and exit of the function here...
-			print(f"Operation Name mocked: {operation_name}\n"
-			      f"Key Name: {test_key}\n"
-			      f"kwargs: {kwargs}\n"
-			      f"mocked return_response: {test_value}")
+			logging.info(f"Operation Name mocked: {operation_name}\n"
+			             f"Key Name: {test_key}\n"
+			             f"kwargs: {kwargs}\n"
+			             f"mocked return_response: {test_value}")
 			return test_value
 
 		return_response = orig(self, operation_name, kwargs)
-		print(f"Operation Name passed through: {operation_name}\n"
-		      f"Key name: {test_key}\n"
-		      f"kwargs: {kwargs}\n"
-		      f"Actual return response: {return_response}")
+		logging.info(f"Operation Name passed through: {operation_name}\n"
+		             f"Key name: {test_key}\n"
+		             f"kwargs: {kwargs}\n"
+		             f"Actual return response: {return_response}")
 		return return_response
 
 	mocker.patch('botocore.client.BaseClient._make_api_call', new=amend_make_api_call)
 
 
-def _amend_make_api_call(meta_key_dict, test_dict, mocker):
+def _amend_make_api_call(meta_key_dict, test_dict, verbosity, mocker):
+	import logging
+	logging.basicConfig(level=verbosity, format="[%(filename)s:%(lineno)s - %(funcName)20s() ] %(message)s")
+
 	orig = client.BaseClient._make_api_call
 
 	def amend_make_api_call(self, operation_name, kwargs):
@@ -112,25 +121,28 @@ def amend_make_api_call(self, operation_name, kwargs):
 				if isinstance(test_value, Exception):
 					# Implied break and exit of the function here...
 					raise test_value
-				print(f"Operation Name mocked: {operation_name}\n"
-				      f"Function Name: {meta_key_dict['FunctionName']}\n"
-				      f"kwargs: {kwargs}\n"
-				      f"mocked return_response: {op_name['test_result']}")
+				logging.info(f"Operation Name mocked: {operation_name}\n"
+				             f"Function Name: {meta_key_dict['FunctionName']}\n"
+				             f"kwargs: {kwargs}\n"
+				             f"mocked return_response: {op_name['test_result']}")
 				return op_name['test_result']
 		try:
-			print(f"Trying: Operation Name passed through: {operation_name}\n"
-			      f"Key Name: {meta_key_dict['FunctionName']}\n"
-			      f"kwargs: {kwargs}\n")
+			logging.info(f"Trying: Operation Name passed through: {operation_name}\n"
+			             f"Key Name: {meta_key_dict['FunctionName']}\n"
+			             f"kwargs: {kwargs}\n")
 			return_response = orig(self, operation_name, kwargs)
-			print(f"Actual return_response: {return_response}")
+			logging.info(f"Actual return_response: {return_response}")
 		except Exception as my_Error:
 			raise ConnectionError("Operation Failed")
 		return return_response
 
 	mocker.patch('botocore.client.BaseClient._make_api_call', new=amend_make_api_call)
 
 
-def _amend_make_api_call_specific(meta_key_dict, test_dict, mocker):
+def _amend_make_api_call_specific(meta_key_dict, test_dict, verbosity, mocker):
+	import logging
+	logging.basicConfig(level=verbosity, format="[%(filename)s:%(lineno)s - %(funcName)20s() ] %(message)s")
+
 	orig = client.BaseClient._make_api_call
 
 	def amend_make_api_call(self, operation_name, kwargs):
@@ -152,30 +164,31 @@ def amend_make_api_call(self, operation_name, kwargs):
 						break
 				if test_value is not None and isinstance(test_value, Exception):
 					# Implied break and exit of the function here...
-					print("Expected Error...")
+					logging.info("Expected Error...")
 					raise test_value
 				elif test_value is None:
-					print(f"No test data offered for this credentials in region {region}")
+					logging.info(f"No test data offered for this credentials in region {region}")
 					continue
-				print(f"Operation Name mocked: {operation_name}\n"
-				      f"Function Name: {meta_key_dict['FunctionName']}\n"
-				      f"kwargs: {kwargs}\n"
-				      f"mocked return_response: {test_value}")
+				logging.info(f"Operation Name mocked: {operation_name}\n"
+				             f"Function Name: {meta_key_dict['FunctionName']}\n"
+				             f"kwargs: {kwargs}\n"
+				             f"mocked return_response: {test_value}")
 				return test_value
 
-		print(f"Operation Name passed through: {operation_name}\n"
-		      f"Function Name: {meta_key_dict['FunctionName']}\n"
-		      f"kwargs: {kwargs}\n")
+		logging.info(f"Operation Name passed through: {operation_name}\n"
+		             f"Function Name: {meta_key_dict['FunctionName']}\n"
+		             f"kwargs: {kwargs}\n")
 		return_response = orig(self, operation_name, kwargs)
-		print(f"Actual return_response: {return_response}")
+		logging.info(f"Actual return_response: {return_response}")
 		return return_response
 
 	mocker.patch('botocore.client.BaseClient._make_api_call', new=amend_make_api_call)
-# mocker.patch('botocore.session', new=amend_make_api_call)
 
 
+# mocker.patch('botocore.session', new=amend_make_api_call)
+
 
-def mock_find_all_instances2(creds:dict, region:str):
+def mock_find_all_instances2(creds: dict, region: str):
 	"""
 	This is a mock function that will return a list of all the instances in the region that we're looking for.
 	:param creds: Credentials object, where 'AccountNumber' is the account number of the account that we're looking for.

inventory-scripts/Tests/test_Inventory_Modules.py

@@ -35,11 +35,12 @@ def test_get_all_credentials(parameters, test_value_dict, mocker):
 	pTiming = parameters['pTiming']
 	pRootOnly = parameters['pRootOnly']
 	pRoleList = parameters['pRoleList']
+	verbose = parameters['pverbose']
 	test_data = {'FunctionName'   : 'get_all_credentials',
 	             'AccountSpecific': True,
 	             'RegionSpecific' : True
 	             }
-	_amend_make_api_call(test_data, test_value_dict, mocker)
+	_amend_make_api_call(test_data, test_value_dict, verbose, mocker)
 
 	# if isinstance(test_value, Exception):
 	# 	print("Expected Error...")