Metaflow UI Cloudformation nested stack support.

msavela · msavela · commit bccdbeefc456 · 2021-09-24T13:08:51.000+03:00
diff --git a/aws/cloudformation/README.md b/aws/cloudformation/README.md
@@ -49,3 +49,37 @@ Did you choose to enable "APIBasicAuth" and/or "CustomRole" and are wondering ho
     2. From the AWS Console, navigate to "Services" and select "API Gateway" from "Networking & Content Delivery" (or search for it in the search bar).  Click your API, select "API Keys" from the left side, select the API that corresponds to your Stack name, and click "show" next to "API Key".
 
 - **CustomRole** - This template can create an optional role that can be assumed by users (or applications) that includes limited permissions to only the resources required by Metaflow, including access only to the Amazon S3 bucket, AWS Batch Compute Environment, and Amazon Sagemaker Notebook Instance created by this template.  You will, however, need to modify the trust policy for the role to grant access to the principals (users/roles/accounts) who will assume it, and you'll also need to have your users configure an appropriate role-assumption profile.  The ARN of the Custom Role can be found in the "Output" tab of the CloudFormation stack under `MetaflowUserRoleArn`.  To modify the trust policy to allow new principals, follow the directions [here](https://docs.aws.amazon.com/IAM/latest/UserGuide/roles-managingrole-editing-console.html#roles-managingrole_edit-trust-policy).  Once you've granted access to the principals of your choice, have your users create a new Profile for the AWS CLI that assumes the role ARN by following the directions [here](https://docs.aws.amazon.com/cli/latest/userguide/cli-configure-role.html).
+
+### Optional Metaflow User Interface (`EnableUI` -parameter)
+
+Please note: This section can be ignored if `EnableUI` -parameter is disabled.
+
+User Interface is provided as a CloudFormation Nested Stack, which means that the UI components are separated to its own template and are deployed to separate CloudFormation Stack. All Nested Stack operations should be initiated from the root stack.
+
+While working with Nested Stacks, templates need to be packaged before deployment. For this AWS Command Line Interface can be used. Packaging step is only required when deploying Metaflow User Interface Nested Stack.
+
+Prerequisites for User Interface enabled Nested Stack deployment:
+
+* **AWS Command Line Interface** - Installed and configured.
+* **S3 bucket to upload CloudFormation templates to** - Nested Stacks require templates referred by absolute S3 paths.
+
+Packaging step is required in order to deploy templates with Nested Stacks:
+
+```bash
+$ aws cloudformation package \
+    --s3-bucket manually-created-s3-bucket \
+    --template-file metaflow-cfn-template.yml \
+    --output-template-file metaflow-cfn-template.yml.package
+```
+
+Above package command does the following:
+
+1. Nested Stack template is uploaded to S3 bucket
+    1. Defined by `--s3-bucket` parameter (this bucket should already exist and needs to be created manually)
+2. Root template is transformed so that all local paths are converted to absolute S3 paths
+    1. Input file is defined by `--template-file` parameter
+3. Transformed and packaged root template is placed in local file system defined by `--output-template-file` parameter
+
+Once this package step is completed, you can follow the [AWS CloudFormation Deployment](https://admin-docs.metaflow.org/metaflow-on-aws/deployment-guide/aws-cloudformation-deployment#steps-for-aws-cloudformation-deployment). Please note: instead of using the non-packaged template, you should use the packaged template `metaflow-cfn-template.yml.package` you just created.
+
+Read more about Cloudformation and working with nested stacks [here](https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/using-cfn-nested-stacks.html).
diff --git a/aws/cloudformation/metaflow-cfn-template.yml b/aws/cloudformation/metaflow-cfn-template.yml
@@ -64,6 +64,11 @@ Parameters:
     Default: 'aws'
     AllowedValues: ['aws', 'aws-us-gov']
     Description: 'IAM Partition (Select aws-us-gov for AWS GovCloud, otherwise leave as is)'
+  EnableUI:
+    Type: String
+    Default: 'false'
+    AllowedValues: ['false', 'true']
+    Description: 'Enable Metaflow UI'
 
 Mappings:
   ServiceInfo:
@@ -92,6 +97,7 @@ Conditions:
   EnableRole: !Equals [ !Ref CustomRole, 'true']
   EnableSagemaker: !Equals [ !Ref EnableSagemaker, 'true']
   IsGov: !Equals [ !Ref IAMPartition, 'aws-us-gov']
+  EnableUI: !Equals [ !Ref EnableUI, 'true']
 Resources:
   VPC:
     Type: AWS::EC2::VPC
@@ -1330,6 +1336,38 @@ Resources:
       TimeToLiveSpecification:
         AttributeName: ttl
         Enabled: true
+
+# Metaflow UI Nested stack
+
+  MetaflowUI:
+    Condition: 'EnableUI'
+    DependsOn:
+      - VPC
+      - Subnet1
+      - Subnet2
+      - FargateSecurityGroup
+      - ECSCluster
+      - ECSTaskExecutionRole
+      - MetadataSvcECSTaskRole
+      - RDSMasterInstance
+      - MyRDSSecret
+    Type: AWS::CloudFormation::Stack
+    Properties:
+      TemplateURL: ui/metaflow-ui-cfn-template.yml
+      Parameters:
+        VPCId: !Ref VPC
+        VPCCidrBlock: !GetAtt 'VPC.CidrBlock'
+        SubnetIds: !Join [',', [ !Ref Subnet1, !Ref Subnet2 ]]
+        SecurityGroup: !Ref FargateSecurityGroup
+        ECSClusterId: !Ref ECSCluster
+        ECSTaskExecutionRoleId: !Ref ECSTaskExecutionRole
+        MetadataSvcECSTaskRoleArn: !GetAtt 'MetadataSvcECSTaskRole.Arn'
+        DatabaseHost: !GetAtt 'RDSMasterInstance.Endpoint.Address'
+        DatabasePort: "5432"
+        DatabaseUser: "master"
+        DatabasePassword: !Join ['', ['{{resolve:secretsmanager:', !Ref MyRDSSecret, ':SecretString:password}}' ]]
+        DatabaseName: "metaflow"
+
 Outputs:
   MetaflowDataStoreS3Url:
     Description: Amazon S3 URL for Metaflow DataStore [METAFLOW_DATASTORE_SYSROOT_S3]
diff --git a/aws/cloudformation/ui/metaflow-ui-cfn-template.yml b/aws/cloudformation/ui/metaflow-ui-cfn-template.yml
@@ -0,0 +1,280 @@
+AWSTemplateFormatVersion: "2010-09-09"
+Description: Nested stack for deployment of Metaflow UI (last-updated-date 09/24/2021)
+
+Parameters:
+  VPCId:
+    Type: AWS::EC2::VPC::Id
+    Description: "VPC Id"
+  VPCCidrBlock:
+    Type: String
+    Description: "VPC CIDR block"
+  SubnetIds:
+    Type: List<AWS::EC2::Subnet::Id>
+    Description: "List of VPC subnet Ids"
+  SecurityGroup:
+    Type: AWS::EC2::SecurityGroup::Id
+    Description: "Security group"
+  ECSClusterId:
+    Type: String
+    Description: "ECS cluster Id"
+  ECSTaskExecutionRoleId:
+    Type: String
+    Description: "ECS task execution IAM role Id"
+  MetadataSvcECSTaskRoleArn:
+    Type: String
+    Description: "ECS service IAM role ARN"
+  DatabaseHost:
+    Type: String
+    Description: "Database hostname"
+  DatabasePort:
+    Type: String
+    Default: "5432"
+    Description: "Database port"
+  DatabaseUser:
+    Type: String
+    Default: "master"
+    Description: "Database user"
+  DatabasePassword:
+    Type: String
+    Description: "Database password"
+  DatabaseName:
+    Type: String
+    Default: "metaflow"
+    Description: "Database name"
+
+Mappings:
+  ServiceInfoUI:
+    ServiceName:
+      value: "metaflow-ui-service"
+    ImageUrl:
+      value: "netflixoss/metaflow_metadata_service"
+    ContainerPort:
+      value: 8083
+    ContainerCpu:
+      value: 512
+    ContainerMemory:
+      value: 1024
+    DesiredCount:
+      value: 1
+
+Resources:
+  # Ingress rules
+
+  IngressRuleUIService:
+    Type: "AWS::EC2::SecurityGroupIngress"
+    Properties:
+      Description: "Allow API Calls Internally (UI service)"
+      GroupId: !Ref SecurityGroup
+      IpProtocol: tcp
+      FromPort: !FindInMap ["ServiceInfoUI", "ContainerPort", "value"]
+      ToPort: !FindInMap ["ServiceInfoUI", "ContainerPort", "value"]
+      CidrIp: !Ref VPCCidrBlock
+
+  # Load balancer
+
+  ALBUISecurityGroup:
+    Type: AWS::EC2::SecurityGroup
+    Properties:
+      GroupDescription: Access to the public facing load balancer
+      VpcId: !Ref VPCId
+      SecurityGroupIngress:
+        # Allow access to ALB from anywhere on the internet
+        - CidrIp: 0.0.0.0/0
+          IpProtocol: -1
+  ALBUI:
+    Type: AWS::ElasticLoadBalancingV2::LoadBalancer
+    Properties:
+      Scheme: internet-facing
+      Type: application
+      Subnets: !Ref SubnetIds
+      SecurityGroups: [!Ref ALBUISecurityGroup]
+  ALBUIListener:
+    Type: AWS::ElasticLoadBalancingV2::Listener
+    DependsOn:
+      - ALBUI
+    Properties:
+      DefaultActions:
+        - TargetGroupArn: !Ref ALBUITargetGroupUIService
+          Type: "forward"
+      LoadBalancerArn: !Ref ALBUI
+      Port: 80
+      Protocol: HTTP
+  ALBUITargetGroupUIService:
+    Type: AWS::ElasticLoadBalancingV2::TargetGroup
+    Properties:
+      HealthCheckIntervalSeconds: 6
+      HealthCheckPath: /ping
+      HealthCheckProtocol: HTTP
+      HealthCheckTimeoutSeconds: 5
+      HealthyThresholdCount: 2
+      TargetType: ip
+      Port: !FindInMap ["ServiceInfoUI", "ContainerPort", "value"]
+      Protocol: HTTP
+      UnhealthyThresholdCount: 2
+      VpcId: !Ref VPCId
+
+  # Coudfront distribution
+
+  CloudFrontDistribution:
+    Type: AWS::CloudFront::Distribution
+    DependsOn:
+      - ALBUI
+    Properties:
+      DistributionConfig:
+        Origins:
+          - DomainName: !GetAtt "ALBUI.DNSName"
+            Id: !Ref ALBUI
+            CustomOriginConfig:
+              HTTPPort: 80
+              OriginProtocolPolicy: http-only
+              OriginKeepaliveTimeout: 60
+              OriginReadTimeout: 30
+              OriginSSLProtocols:
+                - TLSv1
+                - TLSv1.1
+                - TLSv1.2
+                - SSLv3
+        Enabled: true
+        HttpVersion: http2
+        DefaultCacheBehavior:
+          AllowedMethods:
+            - GET
+            - HEAD
+            - DELETE
+            - OPTIONS
+            - PATCH
+            - POST
+            - PUT
+          Compress: true
+          DefaultTTL: 0
+          MinTTL: 0
+          MaxTTL: 0
+          SmoothStreaming: false
+          TargetOriginId: !Ref ALBUI
+          ForwardedValues:
+            QueryString: true
+            Cookies:
+              Forward: none
+          ViewerProtocolPolicy: redirect-to-https
+        CacheBehaviors:
+          - AllowedMethods:
+              - GET
+              - HEAD
+              - DELETE
+              - OPTIONS
+              - PATCH
+              - POST
+              - PUT
+            Compress: false
+            PathPattern: /api/*
+            DefaultTTL: 0
+            MinTTL: 0
+            MaxTTL: 0
+            SmoothStreaming: false
+            TargetOriginId: !Ref ALBUI
+            ForwardedValues:
+              QueryString: true
+              Cookies:
+                Forward: none
+            ViewerProtocolPolicy: redirect-to-https
+        PriceClass: PriceClass_All
+        IPV6Enabled: true
+
+  # ESC Cluster task definition & service
+
+  TaskDefinitionUIService:
+    Type: AWS::ECS::TaskDefinition
+    Properties:
+      Family: !FindInMap ["ServiceInfoUI", "ServiceName", "value"]
+      Cpu: !FindInMap ["ServiceInfoUI", "ContainerCpu", "value"]
+      Memory: !FindInMap ["ServiceInfoUI", "ContainerMemory", "value"]
+      NetworkMode: awsvpc
+      RequiresCompatibilities:
+        - FARGATE
+      ExecutionRoleArn: !Ref ECSTaskExecutionRoleId
+      TaskRoleArn: !Ref MetadataSvcECSTaskRoleArn
+      ContainerDefinitions:
+        - Name: !FindInMap ["ServiceInfoUI", "ServiceName", "value"]
+          Environment:
+            - Name: "MF_METADATA_DB_HOST"
+              Value: !Ref DatabaseHost
+            - Name: "MF_METADATA_DB_PORT"
+              Value: !Ref DatabasePort
+            - Name: "MF_METADATA_DB_USER"
+              Value: !Ref DatabaseUser
+            - Name: "MF_METADATA_DB_PSWD"
+              Value: !Ref DatabasePassword
+            - Name: "MF_METADATA_DB_NAME"
+              Value: !Ref DatabaseName
+            - Name: "UI_ENABLED"
+              Value: "1"
+          Cpu: !FindInMap ["ServiceInfoUI", "ContainerCpu", "value"]
+          Memory: !FindInMap ["ServiceInfoUI", "ContainerMemory", "value"]
+          Image: !FindInMap ["ServiceInfoUI", "ImageUrl", "value"]
+          Command:
+            [
+              "/opt/latest/bin/python3",
+              "-m",
+              "services.ui_backend_service.ui_server",
+            ]
+          PortMappings:
+            - ContainerPort:
+                !FindInMap ["ServiceInfoUI", "ContainerPort", "value"]
+          LogConfiguration:
+            LogDriver: awslogs
+            Options:
+              awslogs-group:
+                !Join [
+                  "",
+                  [
+                    "/ecs/",
+                    !Ref "AWS::StackName",
+                    "-",
+                    !FindInMap ["ServiceInfoUI", "ServiceName", "value"],
+                  ],
+                ]
+              awslogs-region: !Ref "AWS::Region"
+              awslogs-stream-prefix: "ecs"
+  ServiceLogGroupUIService:
+    Type: AWS::Logs::LogGroup
+    Properties:
+      LogGroupName:
+        !Join [
+          "",
+          [
+            "/ecs/",
+            !Ref "AWS::StackName",
+            "-",
+            !FindInMap ["ServiceInfoUI", "ServiceName", "value"],
+          ],
+        ]
+  ECSFargateServiceUIService:
+    Type: AWS::ECS::Service
+    DependsOn: ALBUIListener
+    Properties:
+      ServiceName: !FindInMap ["ServiceInfoUI", "ServiceName", "value"]
+      Cluster: !Ref ECSClusterId
+      LaunchType: FARGATE
+      DeploymentConfiguration:
+        MaximumPercent: 200
+        MinimumHealthyPercent: 75
+      DesiredCount: !FindInMap ["ServiceInfoUI", "DesiredCount", "value"]
+      NetworkConfiguration:
+        AwsvpcConfiguration:
+          AssignPublicIp: ENABLED
+          SecurityGroups:
+            - !Ref SecurityGroup
+          Subnets: !Ref SubnetIds
+      TaskDefinition: !Ref TaskDefinitionUIService
+      LoadBalancers:
+        - ContainerName: !FindInMap ["ServiceInfoUI", "ServiceName", "value"]
+          ContainerPort: !FindInMap ["ServiceInfoUI", "ContainerPort", "value"]
+          TargetGroupArn: !Ref ALBUITargetGroupUIService
+
+Outputs:
+  ServiceUrl:
+    Description: "ALB URL for Metadata UI (Open to Public Access)"
+    Value: !Sub "http://${ALBUI.DNSName}/"
+  ServiceCloudfrontUrl:
+    Description: "Cloudfront URL for Metadata UI (Open to Public Access)"
+    Value: !Sub "https://${CloudFrontDistribution.DomainName}/"
