feature(GH-1613)- New policy management types

backend/dataall/core/environment/__init__.py

@@ -1,3 +1,3 @@
 """The central package of the application to work with the environment"""
 
-from dataall.core.environment import api, cdk, tasks
+from dataall.core.environment import api, db, cdk, tasks

backend/dataall/core/environment/api/input_types.py

@@ -1,6 +1,6 @@
 from dataall.base.api import gql
 from dataall.base.api.constants import GraphQLEnumMapper, SortDirection
-
+from dataall.core.environment.db.environment_enums import PolicyManagementOptions
 
 AwsEnvironmentInput = gql.InputType(
     name='AwsEnvironmentInput',
@@ -101,7 +101,7 @@ class EnvironmentSortField(GraphQLEnumMapper):
         gql.Argument('groupUri', gql.NonNullableType(gql.String)),
         gql.Argument('IAMRoleArn', gql.NonNullableType(gql.String)),
         gql.Argument('environmentUri', gql.NonNullableType(gql.String)),
-        gql.Argument('dataallManaged', gql.NonNullableType(gql.Boolean)),
+        gql.Argument('dataallManaged', gql.NonNullableType(PolicyManagementOptions.toGraphQLEnum())),
     ],
 )
 
@@ -120,5 +120,6 @@ class EnvironmentSortField(GraphQLEnumMapper):
     arguments=[
         gql.Argument('consumptionRoleName', gql.String),
         gql.Argument('groupUri', gql.String),
+        gql.Argument('dataallManaged', gql.NonNullableType(PolicyManagementOptions.toGraphQLEnum())),
     ],
 )

backend/dataall/core/environment/api/types.py

@@ -8,7 +8,7 @@
     resolve_user_role,
 )
 from dataall.core.environment.api.enums import EnvironmentPermission
-
+from dataall.core.environment.db.environment_enums import PolicyManagementOptions
 
 EnvironmentUserPermission = gql.ObjectType(
     name='EnvironmentUserPermission',
@@ -163,7 +163,7 @@
         gql.Field(name='policy_name', type=gql.String),
         gql.Field(name='policy_type', type=gql.String),
         gql.Field(name='exists', type=gql.Boolean),
-        gql.Field(name='attached', type=gql.Boolean),
+        gql.Field(name='attached', type=gql.String),
     ],
 )
 
@@ -176,7 +176,7 @@
         gql.Field(name='environmentUri', type=gql.String),
         gql.Field(name='IAMRoleArn', type=gql.String),
         gql.Field(name='IAMRoleName', type=gql.String),
-        gql.Field(name='dataallManaged', type=gql.Boolean),
+        gql.Field(name='dataallManaged', type=gql.NonNullableType(PolicyManagementOptions.toGraphQLEnum())),
         gql.Field(name='created', type=gql.String),
         gql.Field(name='updated', type=gql.String),
         gql.Field(name='deleted', type=gql.String),

backend/dataall/core/environment/cdk/environment_stack.py

@@ -451,32 +451,33 @@ def create_group_environment_role(self, group: EnvironmentGroup, id: str):
             permissions=group_permissions,
         ).generate_policies()
 
-        external_managed_policies = []
-        policy_manager = PolicyManager(
-            environmentUri=self._environment.environmentUri,
-            resource_prefix=self._environment.resourcePrefix,
-            role_name=group.environmentIAMRoleName,
-            account=self._environment.AwsAccountId,
-            region=self._environment.region,
-        )
-        try:
-            managed_policies = policy_manager.get_all_policies()
-        except Exception as e:
-            logger.info(f'Not adding any managed policy because of exception: {e}')
-            # Known exception raised in first deployment because pivot role does not exist and cannot be assumed
-            managed_policies = []
-        for policy in managed_policies:
-            # If there is a managed policy that exist it should be attached to the IAM role
-            if policy.get('exists', False):
-                external_managed_policies.append(
-                    iam.ManagedPolicy.from_managed_policy_name(
-                        self,
-                        id=f'{self._environment.resourcePrefix}-managed-policy-{policy.get("policy_name")}',
-                        managed_policy_name=policy.get('policy_name'),
+        with self.engine.scoped_session() as session:
+            external_managed_policies = []
+            policy_manager = PolicyManager(
+                session=session,
+                account=self._environment.AwsAccountId,
+                region=self._environment.region,
+                environmentUri=self._environment.environmentUri,
+                resource_prefix=self._environment.resourcePrefix,
+                role_name=group.environmentIAMRoleName,
+            )
+            try:
+                managed_policies = policy_manager.get_all_policies()
+            except Exception as e:
+                logger.info(f'Not adding any managed policy because of exception: {e}')
+                # Known exception raised in first deployment because pivot role does not exist and cannot be assumed
+                managed_policies = []
+            for policy in managed_policies:
+                # If there is a managed policy that exist it should be attached to the IAM role
+                if policy.get('exists', False):
+                    external_managed_policies.append(
+                        iam.ManagedPolicy.from_managed_policy_name(
+                            self,
+                            id=f'{self._environment.resourcePrefix}-managed-policy-{policy.get("policy_name")}',
+                            managed_policy_name=policy.get('policy_name'),
+                        )
                     )
-                )
 
-        with self.engine.scoped_session() as session:
             data_policy = S3Policy(
                 stack=self,
                 tag_key='Team',

backend/dataall/core/environment/db/__init__.py

@@ -0,0 +1 @@
+from . import environment_enums, environment_models, environment_repositories

backend/dataall/core/environment/db/environment_enums.py

@@ -0,0 +1,7 @@
+from dataall.base.api import GraphQLEnumMapper
+
+
+class PolicyManagementOptions(GraphQLEnumMapper):
+    FULLY_MANAGED = 'Fully-Managed'
+    PARTIALLY_MANAGED = 'Partially-Managed'
+    EXTERNALLY_MANAGED = 'Externally-Managed'

backend/dataall/core/environment/db/environment_models.py

@@ -2,7 +2,7 @@
 
 import datetime
 
-from sqlalchemy import Boolean, Column, DateTime, String, ForeignKey
+from sqlalchemy import Boolean, Column, DateTime, String, ForeignKey, Enum
 from sqlalchemy.orm import query_expression
 from dataall.base.db import Resource, Base, utils
 
@@ -107,7 +107,7 @@ class ConsumptionRole(Base):
     groupUri = Column(String, nullable=False)
     IAMRoleName = Column(String, nullable=False)
     IAMRoleArn = Column(String, nullable=False)
-    dataallManaged = Column(Boolean, nullable=False, default=True)
+    dataallManaged = Column(String, nullable=False)
     created = Column(DateTime, default=datetime.datetime.now)
     updated = Column(DateTime, onupdate=datetime.datetime.now)
     deleted = Column(DateTime)

backend/dataall/core/environment/db/environment_repositories.py

@@ -160,7 +160,6 @@ def query_user_environment_consumption_roles(session, groups, uri, filter) -> Qu
                 )
             )
         if filter and filter.get('groupUri'):
-            print('filter group')
             group = filter['groupUri']
             query = query.filter(
                 or_(

backend/dataall/core/environment/services/environment_service.py

@@ -13,6 +13,7 @@
 from dataall.base.aws.sts import SessionHelper
 from dataall.base.context import get_context
 from dataall.base.db.exceptions import AWSResourceNotFound
+from dataall.core.environment.db.environment_enums import PolicyManagementOptions
 from dataall.core.organizations.db.organization_repositories import OrganizationRepository
 from dataall.core.permissions.services.environment_permissions import (
     ENABLE_ENVIRONMENT_SUBSCRIPTIONS,
@@ -46,6 +47,7 @@
 from dataall.core.permissions.services.tenant_permissions import MANAGE_ENVIRONMENTS
 from dataall.core.stacks.db.stack_repositories import StackRepository
 from dataall.core.vpc.db.vpc_repositories import VpcRepository
+from dataall.modules.shares_base.services.shares_enums import PrincipalType
 
 log = logging.getLogger(__name__)
 
@@ -397,15 +399,21 @@ def invite_group(uri, data=None) -> (Environment, EnvironmentGroup):
                 env_group_iam_role_arn = f'arn:aws:iam::{environment.AwsAccountId}:role/{env_group_iam_role_name}'
                 env_role_imported = False
 
-            # If environment role is imported, then data.all should attach the policies at import time
-            # If environment role is created in environment stack, then data.all should attach the policies in the env stack
+            # If environment role is imported, then data.all should attach the policies at import time ( Fully Managed )
+            # If environment role is created in environment stack, then data.all should attach the policies in the env stack ( Partially Managed - Here policy will be created but won't be attached )
+            policy_management: str = (
+                PolicyManagementOptions.FULLY_MANAGED.value
+                if env_role_imported is True
+                else PolicyManagementOptions.PARTIALLY_MANAGED.value
+            )
             PolicyManager(
-                role_name=env_group_iam_role_name,
-                environmentUri=environment.environmentUri,
+                session=session,
                 account=environment.AwsAccountId,
                 region=environment.region,
+                environmentUri=environment.environmentUri,
                 resource_prefix=environment.resourcePrefix,
-            ).create_all_policies(managed=env_role_imported)
+                role_name=env_group_iam_role_name,
+            ).create_all_policies(policy_management=policy_management)
 
             athena_workgroup = NamingConventionService(
                 target_uri=environment.environmentUri,
@@ -470,11 +478,12 @@ def remove_group(uri, group):
             group_membership = EnvironmentService.find_environment_group(session, group, environment.environmentUri)
 
             PolicyManager(
-                role_name=group_membership.environmentIAMRoleName,
-                environmentUri=environment.environmentUri,
+                session=session,
                 account=environment.AwsAccountId,
                 region=environment.region,
+                environmentUri=environment.environmentUri,
                 resource_prefix=environment.resourcePrefix,
+                role_name=group_membership.environmentIAMRoleName,
             ).delete_all_policies()
 
             if group_membership:
@@ -590,16 +599,17 @@ def add_consumption_role(uri, data=None) -> (Environment, EnvironmentGroup):
                 groupUri=group,
                 IAMRoleArn=IAMRoleArn,
                 IAMRoleName=IAMRoleArn.split('/')[-1],
-                dataallManaged=data.get('dataallManaged', True),
+                dataallManaged=data.get('dataallManaged'),
             )
 
             PolicyManager(
-                role_name=consumption_role.IAMRoleName,
-                environmentUri=environment.environmentUri,
+                session=session,
                 account=environment.AwsAccountId,
                 region=environment.region,
+                environmentUri=environment.environmentUri,
                 resource_prefix=environment.resourcePrefix,
-            ).create_all_policies(managed=consumption_role.dataallManaged)
+                role_name=consumption_role.IAMRoleName,
+            ).create_all_policies(policy_management=consumption_role.dataallManaged)
 
             session.add(consumption_role)
             session.commit()
@@ -630,11 +640,12 @@ def remove_consumption_role(uri, env_uri):
 
             if consumption_role:
                 PolicyManager(
-                    role_name=consumption_role.IAMRoleName,
-                    environmentUri=environment.environmentUri,
+                    session=session,
                     account=environment.AwsAccountId,
                     region=environment.region,
+                    environmentUri=environment.environmentUri,
                     resource_prefix=environment.resourcePrefix,
+                    role_name=consumption_role.IAMRoleName,
                 ).delete_all_policies()
 
                 ResourcePolicyService.delete_resource_policy(
@@ -668,6 +679,26 @@ def update_consumption_role(uri, env_uri, input):
                 for key, value in input.items():
                     setattr(consumption_role, key, value)
                 session.commit()
+
+                # If the input consumption role is not Fully-Managed then attach the share policy if it exists
+                if consumption_role.dataallManaged == PolicyManagementOptions.FULLY_MANAGED.value:
+                    environment: Environment = EnvironmentService.get_environment_by_uri(session, env_uri)
+                    share_policy_manager = PolicyManager(
+                        session=session,
+                        account=environment.AwsAccountId,
+                        region=environment.region,
+                        environmentUri=environment.environmentUri,
+                        resource_prefix=environment.resourcePrefix,
+                        role_name=consumption_role.IAMRoleName,
+                    )
+                    for policy_manager in [
+                        Policy
+                        for Policy in share_policy_manager.initializedPolicies
+                        if Policy.policy_type == 'SharePolicy'
+                    ]:
+                        managed_policy_list = policy_manager.get_policies_unattached_to_role()
+                        policy_manager.attach_policies(managed_policy_list)
+
             return consumption_role
 
     @staticmethod
@@ -808,6 +839,15 @@ def paginated_all_environment_consumption_roles(uri, data=None) -> dict:
     def get_consumption_role(session, uri) -> Query:
         return EnvironmentRepository.get_consumption_role(session, uri)
 
+    @staticmethod
+    def get_role_policy_management_type(principal_type: str, principal_id: str):
+        with get_context().db_engine.scoped_session() as session:
+            if principal_type == PrincipalType.ConsumptionRole.value:
+                consumption_role: ConsumptionRole = EnvironmentService.get_consumption_role(session, uri=principal_id)
+                return consumption_role.dataallManaged
+
+        return PolicyManagementOptions.FULLY_MANAGED.value
+
     @staticmethod
     @ResourcePolicyService.has_resource_permission(environment_permissions.LIST_ENVIRONMENT_NETWORKS)
     def paginated_environment_networks(uri, data=None) -> dict:
@@ -893,11 +933,12 @@ def delete_environment(uri):
                     StackStatus.DELETE_COMPLETE.value,
                 ]:
                     PolicyManager(
-                        role_name=environment.EnvironmentDefaultIAMRoleName,
-                        environmentUri=environment.environmentUri,
+                        session=session,
                         account=environment.AwsAccountId,
                         region=environment.region,
+                        environmentUri=environment.environmentUri,
                         resource_prefix=environment.resourcePrefix,
+                        role_name=environment.EnvironmentDefaultIAMRoleName,
                     ).delete_all_policies()
 
                 KeyValueTagRepository.delete_key_value_tags(session, environment.environmentUri, 'environment')
@@ -1111,16 +1152,12 @@ def get_template_from_resource_bucket(uri, template_name):
     @ResourcePolicyService.has_resource_permission(environment_permissions.GET_ENVIRONMENT)
     def resolve_consumption_role_policies(uri, IAMRoleName):
         environment = EnvironmentService.find_environment_by_uri(uri=uri)
-        return PolicyManager(
-            role_name=IAMRoleName,
-            environmentUri=uri,
-            account=environment.AwsAccountId,
-            region=environment.region,
-            resource_prefix=environment.resourcePrefix,
-        ).get_all_policies()
-
-    @staticmethod
-    @ResourcePolicyService.has_resource_permission(environment_permissions.GET_ENVIRONMENT)
-    def get_consumption_role_by_name(uri, IAMRoleName):
         with get_context().db_engine.scoped_session() as session:
-            return EnvironmentRepository.get_environment_consumption_role_by_name(session, uri, IAMRoleName)
+            return PolicyManager(
+                session=session,
+                account=environment.AwsAccountId,
+                region=environment.region,
+                environmentUri=uri,
+                resource_prefix=environment.resourcePrefix,
+                role_name=IAMRoleName,
+            ).get_all_policies()