Merge pull request #42 from msavela/ui-support

savingoyal · web-flow · commit 61deb9580605 · 2021-10-27T05:04:12.000-07:00
Metaflow UI Cloudformation stack support.
diff --git a/aws/cloudformation/README.md b/aws/cloudformation/README.md
@@ -49,3 +49,16 @@ Did you choose to enable "APIBasicAuth" and/or "CustomRole" and are wondering ho
     2. From the AWS Console, navigate to "Services" and select "API Gateway" from "Networking & Content Delivery" (or search for it in the search bar).  Click your API, select "API Keys" from the left side, select the API that corresponds to your Stack name, and click "show" next to "API Key".
 
 - **CustomRole** - This template can create an optional role that can be assumed by users (or applications) that includes limited permissions to only the resources required by Metaflow, including access only to the Amazon S3 bucket, AWS Batch Compute Environment, and Amazon Sagemaker Notebook Instance created by this template.  You will, however, need to modify the trust policy for the role to grant access to the principals (users/roles/accounts) who will assume it, and you'll also need to have your users configure an appropriate role-assumption profile.  The ARN of the Custom Role can be found in the "Output" tab of the CloudFormation stack under `MetaflowUserRoleArn`.  To modify the trust policy to allow new principals, follow the directions [here](https://docs.aws.amazon.com/IAM/latest/UserGuide/roles-managingrole-editing-console.html#roles-managingrole_edit-trust-policy).  Once you've granted access to the principals of your choice, have your users create a new Profile for the AWS CLI that assumes the role ARN by following the directions [here](https://docs.aws.amazon.com/cli/latest/userguide/cli-configure-role.html).
+
+### Optional Metaflow User Interface (`EnableUI` -parameter)
+
+Please note: This section can be ignored if `EnableUI` -parameter is disabled (this is the default value).
+
+User Interface is provided as part of the `metaflow-cfn-template.yml` template and doesn't require any additional
+configuration besides enabling the `EnableUI` -parameter. You can follow the [AWS CloudFormation Deployment](https://admin-docs.metaflow.org/metaflow-on-aws/deployment-guide/aws-cloudformation-deployment#steps-for-aws-cloudformation-deployment) instructions.
+
+Once deployed the Cloudformation Stack will provide two outputs:
+- `UIServiceUrl` - Application Load Balancer endpoint
+- `UIServiceCloudfrontUrl` - Cloudfront distribution (using ALB) endpoint with HTTPS enabled (preferred)
+
+Please note: Metaflow User Interface doesn't provide any authentication by default.
diff --git a/aws/cloudformation/metaflow-cfn-template.yml b/aws/cloudformation/metaflow-cfn-template.yml
@@ -1,5 +1,5 @@
 AWSTemplateFormatVersion: '2010-09-09'
-Description: Stack for complete deployment of Metaflow (last-updated-date 02/09/2021)
+Description: Stack for complete deployment of Metaflow (last-updated-date 10/26/2021)
 
 Parameters:
   SagemakerInstance:
@@ -73,6 +73,11 @@ Parameters:
     Type: String
     Default: ''
     Description: 'Additional IAM Policy ARN to attach to Batch Compute Environment (leave empty, unless you know what you are doing)'
+  EnableUI:
+    Type: String
+    Default: 'false'
+    AllowedValues: ['false', 'true']
+    Description: 'Enable Metaflow UI'
 
 Mappings:
   ServiceInfo:
@@ -96,13 +101,29 @@ Mappings:
       value: 1
     Role:
       value: ""
+
+  ServiceInfoUI:
+    ServiceName:
+      value: 'metaflow-ui-service'
+    ImageUrl:
+      value: 'netflixoss/metaflow_metadata_service'
+    ContainerPort:
+      value: 8083
+    ContainerCpu:
+      value: 4096
+    ContainerMemory:
+      value: 16384
+    DesiredCount:
+      value: 1
+
 Conditions:
   EnableAuth: !Equals [ !Ref APIBasicAuth, 'true']
   EnableRole: !Equals [ !Ref CustomRole, 'true']
   EnableFargateOnBatch: !Equals [ !Ref BatchType, 'fargate']
   EnableSagemaker: !Equals [ !Ref EnableSagemaker, 'true']
   IsGov: !Equals [ !Ref IAMPartition, 'aws-us-gov']
   EnableAddtionalWorkerPolicy: !Not [ !Equals [ !Ref AdditionalWorkerPolicyArn, ''] ]
+  EnableUI: !Equals [ !Ref EnableUI, 'true']
 Resources:
   VPC:
     Type: AWS::EC2::VPC
@@ -294,11 +315,16 @@ Resources:
           PolicyDocument:
             Version: '2012-10-17'
             Statement:
-              Sid: ObjectAccessMetadataService
-              Effect: Allow
-              Action:
-                - s3:GetObject
-              Resource: !Join ['', [ !GetAtt 'MetaflowS3Bucket.Arn', '/*' ]]
+              - Sid: ObjectAccessMetadataService
+                Effect: Allow
+                Action:
+                  - s3:GetObject
+                Resource: !Join ['', [ !GetAtt 'MetaflowS3Bucket.Arn', '/*' ]]
+              - Sid: ObjectAccessMetadataServiceNonExistentKeys
+                Effect: Allow
+                Action:
+                  - s3:ListBucket
+                Resource: !GetAtt 'MetaflowS3Bucket.Arn'
         - PolicyName: DenyPresignedBatch
           PolicyDocument:
             Version: '2012-10-17'
@@ -1357,6 +1383,237 @@ Resources:
       TimeToLiveSpecification:
         AttributeName: ttl
         Enabled: true
+
+# --- Metaflow UI resources begin ---
+
+  # UI: Ingress rules
+
+  IngressRuleUIService:
+    Type: AWS::EC2::SecurityGroupIngress
+    Condition: EnableUI
+    Properties:
+      Description: "Allow API Calls Internally (UI service)"
+      GroupId: !Ref FargateSecurityGroup
+      IpProtocol: tcp
+      FromPort: !FindInMap ["ServiceInfoUI", "ContainerPort", "value"]
+      ToPort: !FindInMap ["ServiceInfoUI", "ContainerPort", "value"]
+      CidrIp: !GetAtt 'VPC.CidrBlock'
+
+  # UI: Load balancer
+
+  ALBUISecurityGroup:
+    Type: AWS::EC2::SecurityGroup
+    Condition: EnableUI
+    Properties:
+      GroupDescription: "Access to the public facing load balancer"
+      VpcId: !Ref VPC
+      SecurityGroupIngress:
+        # Allow access to ALB from anywhere on the internet
+        - CidrIp: 0.0.0.0/0
+          IpProtocol: -1
+  ALBUI:
+    Type: AWS::ElasticLoadBalancingV2::LoadBalancer
+    Condition: EnableUI
+    Properties:
+      Scheme: internet-facing
+      Type: application
+      Subnets:
+        - !Ref Subnet1
+        - !Ref Subnet2
+      SecurityGroups: [!Ref ALBUISecurityGroup]
+  ALBUIListener:
+    Type: AWS::ElasticLoadBalancingV2::Listener
+    Condition: EnableUI
+    DependsOn:
+      - ALBUI
+    Properties:
+      DefaultActions:
+        - TargetGroupArn: !Ref ALBUITargetGroupUIService
+          Type: "forward"
+      LoadBalancerArn: !Ref ALBUI
+      Port: 80
+      Protocol: HTTP
+  ALBUITargetGroupUIService:
+    Type: AWS::ElasticLoadBalancingV2::TargetGroup
+    Condition: EnableUI
+    Properties:
+      HealthCheckIntervalSeconds: 6
+      HealthCheckPath: /ping
+      HealthCheckProtocol: HTTP
+      HealthCheckTimeoutSeconds: 5
+      HealthyThresholdCount: 2
+      TargetType: ip
+      Port: !FindInMap ["ServiceInfoUI", "ContainerPort", "value"]
+      Protocol: HTTP
+      UnhealthyThresholdCount: 2
+      VpcId: !Ref VPC
+
+  # UI: Coudfront distribution
+
+  CloudFrontDistributionUI:
+    Type: AWS::CloudFront::Distribution
+    Condition: EnableUI
+    DependsOn:
+      - ALBUI
+    Properties:
+      DistributionConfig:
+        Origins:
+          - DomainName: !GetAtt "ALBUI.DNSName"
+            Id: !Ref ALBUI
+            CustomOriginConfig:
+              HTTPPort: 80
+              OriginProtocolPolicy: http-only
+              OriginKeepaliveTimeout: 60
+              OriginReadTimeout: 30
+              OriginSSLProtocols:
+                - TLSv1
+                - TLSv1.1
+                - TLSv1.2
+                - SSLv3
+        Enabled: true
+        HttpVersion: http2
+        DefaultCacheBehavior:
+          AllowedMethods:
+            - GET
+            - HEAD
+            - DELETE
+            - OPTIONS
+            - PATCH
+            - POST
+            - PUT
+          Compress: true
+          DefaultTTL: 0
+          MinTTL: 0
+          MaxTTL: 0
+          SmoothStreaming: false
+          TargetOriginId: !Ref ALBUI
+          ForwardedValues:
+            QueryString: true
+            Cookies:
+              Forward: none
+          ViewerProtocolPolicy: redirect-to-https
+        CacheBehaviors:
+          - AllowedMethods:
+              - GET
+              - HEAD
+              - DELETE
+              - OPTIONS
+              - PATCH
+              - POST
+              - PUT
+            Compress: false
+            PathPattern: /api/*
+            DefaultTTL: 0
+            MinTTL: 0
+            MaxTTL: 0
+            SmoothStreaming: false
+            TargetOriginId: !Ref ALBUI
+            ForwardedValues:
+              QueryString: true
+              Cookies:
+                Forward: none
+            ViewerProtocolPolicy: redirect-to-https
+        PriceClass: PriceClass_All
+        IPV6Enabled: true
+
+  # UI: ECS Cluster task definition & service
+
+  TaskDefinitionUIService:
+    Type: AWS::ECS::TaskDefinition
+    Condition: EnableUI
+    Properties:
+      Family: !FindInMap ["ServiceInfoUI", "ServiceName", "value"]
+      Cpu: !FindInMap ["ServiceInfoUI", "ContainerCpu", "value"]
+      Memory: !FindInMap ["ServiceInfoUI", "ContainerMemory", "value"]
+      NetworkMode: awsvpc
+      RequiresCompatibilities:
+        - FARGATE
+      ExecutionRoleArn: !Ref 'ECSTaskExecutionRole'
+      TaskRoleArn: !GetAtt 'MetadataSvcECSTaskRole.Arn'
+      ContainerDefinitions:
+        - Name: !FindInMap ["ServiceInfoUI", "ServiceName", "value"]
+          Environment:
+            - Name: "MF_METADATA_DB_HOST"
+              Value: !GetAtt 'RDSMasterInstance.Endpoint.Address'
+            - Name: "MF_METADATA_DB_PORT"
+              Value: "5432"
+            - Name: "MF_METADATA_DB_USER"
+              Value: !Join ['', ['{{resolve:secretsmanager:', !Ref MyRDSSecret, ':SecretString:username}}' ]]
+            - Name: "MF_METADATA_DB_PSWD"
+              Value: !Join ['', ['{{resolve:secretsmanager:', !Ref MyRDSSecret, ':SecretString:password}}' ]]
+            - Name: "MF_METADATA_DB_NAME"
+              Value: "metaflow"
+            - Name: "UI_ENABLED"
+              Value: "1"
+          Cpu: !FindInMap ["ServiceInfoUI", "ContainerCpu", "value"]
+          Memory: !FindInMap ["ServiceInfoUI", "ContainerMemory", "value"]
+          Image: !FindInMap ["ServiceInfoUI", "ImageUrl", "value"]
+          Command:
+            [
+              "/opt/latest/bin/python3",
+              "-m",
+              "services.ui_backend_service.ui_server",
+            ]
+          PortMappings:
+            - ContainerPort:
+                !FindInMap ["ServiceInfoUI", "ContainerPort", "value"]
+          LogConfiguration:
+            LogDriver: awslogs
+            Options:
+              awslogs-group:
+                !Join [
+                  "",
+                  [
+                    "/ecs/",
+                    !Ref "AWS::StackName",
+                    "-",
+                    !FindInMap ["ServiceInfoUI", "ServiceName", "value"],
+                  ],
+                ]
+              awslogs-region: !Ref "AWS::Region"
+              awslogs-stream-prefix: "ecs"
+  ServiceLogGroupUIService:
+    Type: AWS::Logs::LogGroup
+    Condition: EnableUI
+    Properties:
+      LogGroupName:
+        !Join [
+          "",
+          [
+            "/ecs/",
+            !Ref "AWS::StackName",
+            "-",
+            !FindInMap ["ServiceInfoUI", "ServiceName", "value"],
+          ],
+        ]
+  ECSFargateServiceUIService:
+    Type: AWS::ECS::Service
+    Condition: EnableUI
+    DependsOn: ALBUIListener
+    Properties:
+      ServiceName: !FindInMap ["ServiceInfoUI", "ServiceName", "value"]
+      Cluster: !Ref 'ECSCluster'
+      LaunchType: FARGATE
+      DeploymentConfiguration:
+        MaximumPercent: 200
+        MinimumHealthyPercent: 75
+      DesiredCount: !FindInMap ["ServiceInfoUI", "DesiredCount", "value"]
+      NetworkConfiguration:
+        AwsvpcConfiguration:
+          AssignPublicIp: ENABLED
+          SecurityGroups:
+            - !Ref FargateSecurityGroup
+          Subnets:
+            - !Ref Subnet1
+            - !Ref Subnet2
+      TaskDefinition: !Ref TaskDefinitionUIService
+      LoadBalancers:
+        - ContainerName: !FindInMap ["ServiceInfoUI", "ServiceName", "value"]
+          ContainerPort: !FindInMap ["ServiceInfoUI", "ContainerPort", "value"]
+          TargetGroupArn: !Ref ALBUITargetGroupUIService
+
+# --- Metaflow UI resources end ---
+
 Outputs:
   MetaflowDataStoreS3Url:
     Description: Amazon S3 URL for Metaflow DataStore [METAFLOW_DATASTORE_SYSROOT_S3]
@@ -1400,3 +1657,14 @@ Outputs:
   MigrationFunctionName:
     Description: "Name of DB Migration Function"
     Value: !Ref ExecuteDBMigration
+  
+  # Metaflow UI outputs
+
+  UIServiceUrl:
+    Description: "ALB URL for Metadata UI (Open to Public Access)"
+    Value: !Sub "http://${ALBUI.DNSName}/"
+    Condition: EnableUI
+  UIServiceCloudfrontUrl:
+    Description: "Cloudfront URL for Metadata UI (Open to Public Access)"
+    Value: !Sub "https://${CloudFrontDistributionUI.DomainName}/"
+    Condition: EnableUI
