Merge dev into main

Author: DinisCruz

.github/workflows/ci-pipeline__dev.yml

@@ -24,6 +24,8 @@ jobs:
 
       - name: Start Local Stack
         uses: owasp-sbot/OSBot-GitHub-Actions/.github/actions/docker__local-stack@dev
+        with:
+          LOCAL_STACK_SERVICES: 's3,lambda,iam,cloudwatch,dynamodb,logs,sts,ec2'
 
       - name: Poetry - Install Dependencies
         uses: owasp-sbot/OSBot-GitHub-Actions/.github/actions/poetry__install@dev

.github/workflows/ci-pipeline__main.yml

@@ -80,6 +80,7 @@ jobs:
 
 
   publish-to-docker-hub:
+    if: False
     runs-on: ubuntu-latest
 
     steps:

README.md

@@ -1,6 +1,6 @@
 # OSBot-AWS
 
-![Current Release](https://img.shields.io/badge/release-v2.37.0-blue)
+![Current Release](https://img.shields.io/badge/release-v2.37.23-blue)
 
 Large number of AWS apis and utils from the OWASP Security Bot (OSBot) which make
 the use of AWS's boto3 library easier and more intuitive.

osbot_aws/AWS_Config.py

@@ -1,6 +1,4 @@
-import os
-from osbot_utils.base_classes.Type_Safe import Type_Safe
-from osbot_utils.utils.Env              import load_dotenv
+from osbot_utils.type_safe.Type_Safe import Type_Safe
 
 DEFAULT__AWS_DEFAULT_REGION = 'eu-west-1'
 
@@ -12,6 +10,8 @@
 class AWS_Config(Type_Safe):
 
     def __init__(self):
+        from osbot_utils.utils.Env import load_dotenv
+
         super().__init__()
         load_dotenv()
 
@@ -21,29 +21,29 @@ def __enter__(self):
     def __exit__(self, type, value, traceback):
         pass
 
-    def aws_access_key_id           (self): return os.getenv('AWS_ACCESS_KEY_ID'              )
-    def aws_secret_access_key       (self): return os.getenv('AWS_SECRET_ACCESS_KEY'          )
-    def aws_endpoint_url            (self): return os.getenv(ENV_NAME__AWS_ENDPOINT_URL       )
-    def aws_session_profile_name    (self): return os.getenv('AWS_PROFILE_NAME'               )
-    def aws_session_region_name     (self): return os.getenv(ENV_NAME__AWS_DEFAULT_REGION     ) or DEFAULT__AWS_DEFAULT_REGION
-    def aws_session_account_id      (self): return os.getenv(ENV_NAME__AWS_ACCOUNT_ID         ) or  self.sts__session_account_id()
-
-    def dev_skip_aws_key_check      (self): return os.getenv('DEV_SKIP_AWS_KEY_CHECK'        , False              )     # use to not have the 500ms check that happens during this check
-    def bot_name                    (self): return os.getenv('OSBOT_NAME'                                         )     # todo: refactor variable to osbot_name (need to check for side effects)
-    def lambda_s3_folder_layers     (self): return os.getenv('OSBOT_LAMBDA_S3_FOLDER_LAYERS' , 'layers'           )     # todo: change these static values to DEFAULT_.... ones
-    def lambda_s3_folder_lambdas    (self): return os.getenv('OSBOT_LAMBDA_S3_FOLDER_LAMBDAS', 'lambdas'          )
-    def lambda_role_name            (self): return os.getenv('OSBOT_LAMBDA_ROLE_NAME'          'role-osbot-lambda')
-
-    def set_aws_access_key_id       (self, value): os.environ['AWS_ACCESS_KEY_ID'               ] = value ; return value
-    def set_aws_secret_access_key   (self, value): os.environ['AWS_SECRET_ACCESS_KEY'           ] = value ; return value
-    def set_aws_session_profile_name(self, value): os.environ['AWS_PROFILE_NAME'                ] = value ; return value
-    def set_aws_session_region_name (self, value): os.environ[ENV_NAME__AWS_DEFAULT_REGION      ] = value ; return value
-    def set_aws_session_account_id  (self, value): os.environ[ENV_NAME__AWS_ACCOUNT_ID          ] = value ; return value
-    def set_lambda_s3_bucket        (self, value): os.environ['OSBOT_LAMBDA_S3_BUCKET'          ] = value ; return value
-    def set_lambda_s3_folder_layers (self, value): os.environ['OSBOT_LAMBDA_S3_FOLDER_LAYERS'   ] = value ; return value
-    def set_lambda_s3_folder_lambdas(self, value): os.environ['OSBOT_LAMBDA_S3_FOLDER_LAMBDAS'  ] = value ; return value
-    def set_lambda_role_name        (self, value): os.environ['OSBOT_LAMBDA_ROLE_NAME'          ] = value ; return value
-    def set_bot_name                (self, value): os.environ['OSBOT_NAME'                      ] = value ; return value
+    def aws_access_key_id           (self): import os; return os.getenv('AWS_ACCESS_KEY_ID'              )
+    def aws_secret_access_key       (self): import os; return os.getenv('AWS_SECRET_ACCESS_KEY'          )
+    def aws_endpoint_url            (self): import os; return os.getenv(ENV_NAME__AWS_ENDPOINT_URL       )
+    def aws_session_profile_name    (self): import os; return os.getenv('AWS_PROFILE_NAME'               )
+    def aws_session_region_name     (self): import os; return os.getenv(ENV_NAME__AWS_DEFAULT_REGION     ) or DEFAULT__AWS_DEFAULT_REGION
+    def aws_session_account_id      (self): import os; return os.getenv(ENV_NAME__AWS_ACCOUNT_ID         ) or  self.sts__session_account_id()
+
+    def dev_skip_aws_key_check      (self): import os; return os.getenv('DEV_SKIP_AWS_KEY_CHECK'        , False              )     # use to not have the 500ms check that happens during this check
+    def bot_name                    (self): import os; return os.getenv('OSBOT_NAME'                                         )     # todo: refactor variable to osbot_name (need to check for side effects)
+    def lambda_s3_folder_layers     (self): import os; return os.getenv('OSBOT_LAMBDA_S3_FOLDER_LAYERS' , 'layers'           )     # todo: change these static values to DEFAULT_.... ones
+    def lambda_s3_folder_lambdas    (self): import os; return os.getenv('OSBOT_LAMBDA_S3_FOLDER_LAMBDAS', 'lambdas'          )
+    def lambda_role_name            (self): import os; return os.getenv('OSBOT_LAMBDA_ROLE_NAME'          'role-osbot-lambda')
+
+    def set_aws_access_key_id       (self, value): import os; os.environ['AWS_ACCESS_KEY_ID'               ] = value ; return value
+    def set_aws_secret_access_key   (self, value): import os; os.environ['AWS_SECRET_ACCESS_KEY'           ] = value ; return value
+    def set_aws_session_profile_name(self, value): import os; os.environ['AWS_PROFILE_NAME'                ] = value ; return value
+    def set_aws_session_region_name (self, value): import os; os.environ[ENV_NAME__AWS_DEFAULT_REGION      ] = value ; return value
+    def set_aws_session_account_id  (self, value): import os; os.environ[ENV_NAME__AWS_ACCOUNT_ID          ] = value ; return value
+    def set_lambda_s3_bucket        (self, value): import os; os.environ['OSBOT_LAMBDA_S3_BUCKET'          ] = value ; return value
+    def set_lambda_s3_folder_layers (self, value): import os; os.environ['OSBOT_LAMBDA_S3_FOLDER_LAYERS'   ] = value ; return value
+    def set_lambda_s3_folder_lambdas(self, value): import os; os.environ['OSBOT_LAMBDA_S3_FOLDER_LAMBDAS'  ] = value ; return value
+    def set_lambda_role_name        (self, value): import os; os.environ['OSBOT_LAMBDA_ROLE_NAME'          ] = value ; return value
+    def set_bot_name                (self, value): import os; os.environ['OSBOT_NAME'                      ] = value ; return value
 
     def sts__session_account_id(self):                   # to handle when the AWS_ACCOUNT_ID is not set
         #if self.aws_configured():                      # todo: find an efficient way to handle this since this doesn't work when a role is attached to a lambda function or EC2 instance
@@ -73,12 +73,15 @@ def region_name(self):
         return self.aws_session_region_name()
 
     def resolve_lambda_bucket_name(self):
+        import os
         bucket_name = os.getenv('OSBOT_LAMBDA_S3_BUCKET')
         if bucket_name is None:
             bucket_name = f'{self.aws_session_account_id()}--osbot-lambdas--{self.region_name()}' # this is a needed breaking change
         return bucket_name
 
     def resolve_temp_data_bucket_name(self):
+        import os
+
         bucket_name = os.getenv('OSBOT_TEMP_DATA_S3_BUCKET')
         if bucket_name is None:
             bucket_name = f'{self.aws_session_account_id()}--temp-data--{self.region_name()}'

osbot_aws/apis/Session.py

@@ -11,7 +11,6 @@
 from osbot_aws.exceptions.Session_Client_Creation_Fail  import Session_Client_Creation_Fail
 from osbot_aws.exceptions.Session_No_Credentials        import Session_No_Credentials
 
-#ENV_NAME__OSBOT_AWS__SESSION__SET_SIGNATURE_VERSION = 'OSBOT_AWS__SESSION__SET_SIGNATURE_VERSION'      # todo: see if this needed (see comments in default_config method)
 
 class Session(Kwargs_To_Self):                  # todo: refactor to AWS_Session so it doesn't clash with boto3.Session object
     account_id      = None
@@ -26,6 +25,7 @@ def boto_session(self) -> boto3.Session:
 
     @cache
     def botocore_session(self, aws_access_key_id=None , aws_secret_access_key=None , aws_session_token= None, region_name= None, botocore_session=None, profile_name=None ):
+        import boto3
         kwargs = {  "aws_access_key_id"     : aws_access_key_id     ,
                     "aws_secret_access_key" : aws_secret_access_key ,
                     "aws_session_token"     : aws_session_token     ,
@@ -67,6 +67,7 @@ def caller_identity(self, session=None):
 
     @cache_on_self
     def session(self) -> boto3.Session:
+        import boto3
         return boto3.Session()
 
     def session_default(self):
@@ -81,12 +82,12 @@ def client(self, service_name, **kwargs):
             raise Session_Client_Creation_Fail(status=status)
 
     def client_boto3(self,service_name, aws_access_key_id=None, aws_secret_access_key=None, region_name=None, config=None, endpoint_url=None, profile_name=None):                   # todo: refactor add this to resource_boto3
+        import boto3
         try:
             self.config       = config       or self.config       or self.default_config                ()
             self.region_name  = region_name  or self.region_name  or aws_config.aws_session_region_name ()
             self.profile_name = profile_name or self.profile_name or aws_config.aws_session_profile_name()
-            self.endpoint_url = endpoint_url or self.endpoint_url or aws_config.aws_endpoint_url()
-
+            self.endpoint_url = endpoint_url or self.endpoint_url or self.resolve_endpoint_url_for_service(service_name)
 
             client_kwargs = dict( service_name = service_name , config = self.config)
 
@@ -128,10 +129,25 @@ def profiles(self):
 
     def profiles_map(self):
         return get_session()._build_profile_map()
-
         #return self.boto_session()._build_profile_map()
 
+    def resolve_endpoint_url_for_service(self, service):                                   # todo: find a better way to do this mapping to the local_stack services that are supported
+        supported_local_stack_services = ['s3', 'lambda','logs']                           # for now only these 3 services have been tested with local stack , all others will ahve the default
+        endpoint_url = aws_config.aws_endpoint_url()
+        if endpoint_url == 'http://localhost:4566':
+            if service in supported_local_stack_services:
+                return endpoint_url
+            else:
+                region = aws_config.aws_session_region_name()
+                if region:
+                    return f'https://{service}.{region}.amazonaws.com'
+                else:
+                    return None
+        else:
+            return endpoint_url
+
     def resource_boto3(self,service_name, profile_name=None, region_name=None):                 # todo: refactor with client_boto3
+        import boto3
         try:
             profile_name = profile_name or aws_config.aws_session_profile_name()
             region_name  = region_name  or aws_config.aws_session_region_name()
@@ -147,6 +163,7 @@ def resource_boto3(self,service_name, profile_name=None, region_name=None):
     def resource(self, service_name,profile_name=None, region_name=None):
         return self.resource_boto3(service_name,profile_name,region_name).get('resource')
 
+
     # client helpers
 
     # putting all these here in global location so that we limit the calls to authenticate

osbot_aws/apis/shell/Http__Remote_Shell.py

@@ -0,0 +1,39 @@
+import requests
+from osbot_utils.type_safe.Type_Safe    import Type_Safe
+from osbot_utils.utils.Dev              import pprint
+from osbot_aws.apis.shell.Shell_Client  import Shell_Client
+
+class Http__Remote_Shell(Type_Safe, Shell_Client):
+    target_url: str = None
+
+    def _invoke(self, method_name, method_kwargs=None, return_logs=False):
+        auth_key = self._lambda_auth_key()
+
+        payload  = {'lambda_shell': {'method_name'  : method_name    ,
+                                     'method_kwargs': method_kwargs  ,
+                                     'auth_key'     : auth_key       }}
+        response = requests.post(self.target_url, json=payload)
+        if response.headers.get('Content-Type') == 'application/json':
+            return response.json()
+        return response.text.strip()
+
+    def exec_print(self, executable, *params):
+        result = self.exec(executable, params)
+        pprint(result)
+        return result
+
+    def function(self, function):
+        return self.python_exec_function(function)
+
+    def function__print(self,function):
+        result = self.function(function)
+        pprint(result)
+        return result
+
+    # helper execution methods
+
+    def env_vars(self):
+        def return_dict_environ():
+            from os import environ
+            return dict(environ)
+        return self.function(return_dict_environ)

osbot_aws/aws/boto3/View_Boto3_Rest_Calls.py

@@ -1,9 +1,9 @@
 from functools                                  import wraps
 from botocore.client                            import BaseClient
 from osbot_utils.helpers.Print_Table            import Print_Table
+from osbot_utils.helpers.duration.Duration      import Duration
 from osbot_utils.utils.Dev                      import pformat
 from osbot_utils.base_classes.Kwargs_To_Self    import Kwargs_To_Self
-from osbot_utils.testing.Duration               import Duration
 from osbot_utils.testing.Hook_Method            import Hook_Method
 from osbot_utils.utils.Objects                  import obj_data
 

osbot_aws/aws/cloud_front/Cloud_Front.py

@@ -1,6 +1,11 @@
-from osbot_aws.apis.Session import Session
+import time
+from typing                                     import List
+
+from osbot_utils.type_safe.decorators.type_safe import type_safe
+
+from osbot_aws.apis.Session                     import Session
+from osbot_utils.base_classes.Kwargs_To_Self    import Kwargs_To_Self
 
-from osbot_utils.base_classes.Kwargs_To_Self import Kwargs_To_Self
 
 
 class Cloud_Front(Kwargs_To_Self):
@@ -14,3 +19,14 @@ def distributions(self):
         items = response.get('DistributionList', {}).get('Items', [])
         return items
 
+    def invalidate_path(self, distribution_id: str, paths: str):
+        return self.invalidate_paths(distribution_id, [paths])
+
+    @type_safe
+    def invalidate_paths(self, distribution_id:str, paths: List[str]):
+        kwargs = dict(DistributionId    = distribution_id,
+                      InvalidationBatch = { 'Paths'          : { 'Quantity': len(paths)            ,
+                                                                 'Items'   : paths                 },
+                                            'CallerReference': f'invalidation-{int(time.time()) }' })  # Unique reference
+        response = self.client().create_invalidation(**kwargs)
+        return response

osbot_aws/aws/dynamo_db/Dynamo_DB.py

@@ -1,7 +1,4 @@
-import uuid
-from boto3.dynamodb.types                               import TypeDeserializer, TypeSerializer
-from osbot_aws.apis.Session                             import Session
-from osbot_utils.base_classes.Type_Safe                 import Type_Safe
+from osbot_utils.type_safe.Type_Safe                 import Type_Safe
 from osbot_utils.decorators.methods.cache_on_self       import cache_on_self
 from osbot_utils.decorators.methods.remove_return_value import remove_return_value
 
@@ -17,10 +14,14 @@ class Dynamo_DB(Type_Safe):
     # helpers
     @cache_on_self
     def client(self):
+        from osbot_aws.apis.Session import Session
+
         return Session().client(service_name='dynamodb', region_name=self.region_name, endpoint_url=self.endpoint_url)
 
     @cache_on_self
     def client__dynamo_streams(self):
+        from osbot_aws.apis.Session import Session
+
         return Session().client('dynamodbstreams')
 
     # main methods
@@ -43,13 +44,16 @@ def document_delete(self, table_name, key_name, key_value):
         return True                                                #       so unless there was an exception thrown, assume it did
 
     def document_deserialize(self, item):
+        from boto3.dynamodb.types import TypeDeserializer
         if item:
             deserializer = TypeDeserializer()
             return {k: deserializer.deserialize(v) for k, v in item.items()}
         return {}
 
     @remove_return_value('ResponseMetadata')
     def document_update(self, table_name, key_name, key_value, update_data):
+        from boto3.dynamodb.types import TypeSerializer
+
         # Initialize TypeSerializer to convert Python types to DynamoDB types
         serializer = TypeSerializer()
 
@@ -89,6 +93,8 @@ def document_update(self, table_name, key_name, key_value, update_data):
         return response
 
     def document_serialize(self, document):
+        from boto3.dynamodb.types import TypeSerializer
+
         serializer = TypeSerializer()
         return {k: serializer.serialize(v) for k, v in document.items()}
 
@@ -290,4 +296,5 @@ def streams(self):
         return self.client__dynamo_streams().list_streams().get('Streams')
 
     def random_id(self):
+        import uuid
         return str(uuid.uuid4())

osbot_aws/aws/dynamo_db/Dynamo_DB__Table.py

@@ -1,16 +1,17 @@
-import uuid
-
+from osbot_utils.type_safe.Type_Safe                 import Type_Safe
 from osbot_aws.aws.dynamo_db.Dynamo_DB                  import Dynamo_DB
-from osbot_aws.aws.dynamo_db.Dynamo_DB__Record          import Dynamo_DB__Record
-from osbot_utils.base_classes.Kwargs_To_Self            import Kwargs_To_Self
 from osbot_utils.decorators.methods.capture_status      import apply_capture_status
-from osbot_utils.utils.Misc                             import timestamp_utc_now, wait_for
+
 
 PRIMARY_KEY_NAME = 'id'
 PRIMARY_KEY_TYPE = 'S'
 
+from typing import TYPE_CHECKING
+if TYPE_CHECKING:
+    from osbot_aws.aws.dynamo_db.Dynamo_DB__Record          import Dynamo_DB__Record
+
 @apply_capture_status
-class Dynamo_DB__Table(Kwargs_To_Self):
+class Dynamo_DB__Table(Type_Safe):
     dynamo_db  : Dynamo_DB
     key_name   : str
     key_type   : str
@@ -32,12 +33,13 @@ def add_documents(self, documents):
         documents_to_add = []
         for document in documents:
             if self.key_name not in document:                               # If key is present,
-                document = {self.key_name: self.dynamo_db.random_id(),      # add it as a generate a random UUID as the key
+                document = {self.key_name: self.dynamo_db.random_id(),      # add random UUID as the key
                             **document}                                     # to the current document object
             documents_to_add.append(document)
         return self.dynamo_db.documents_add(table_name=self.table_name, documents=documents_to_add)
 
-    def add_record(self, record : Dynamo_DB__Record):
+    def add_record(self, record : 'Dynamo_DB__Record'):
+        from osbot_utils.utils.Misc import timestamp_utc_now
         metadata = record.metadata
         metadata.timestamp_created = timestamp_utc_now()
         document   = record.serialize_to_dict()
@@ -132,6 +134,8 @@ def gsi_delete(self, index_name):
         return self.update_table(gsi_updates=gsi_updates).get('data')
 
     def gsi_wait_for_status(self, status='ACTIVE', max_attempts=20, delay=0.05):        # todo: see if these values need to be higher when dealing with AWS DynamoDB (vs the dynamodb-local)
+        from osbot_utils.utils.Misc import wait_for
+
         for i in range(max_attempts):
             all_match = False
             for gsi in self.gsis().get('data'):