Based on PR comments the nested UI template is now dropped. Everything related to UI is now within the main template.

msavela · msavela · commit da1d2703b2ae · 2021-10-26T18:07:34.000+03:00
diff --git a/aws/cloudformation/README.md b/aws/cloudformation/README.md
@@ -52,35 +52,13 @@ Did you choose to enable "APIBasicAuth" and/or "CustomRole" and are wondering ho
 
 ### Optional Metaflow User Interface (`EnableUI` -parameter)
 
-Please note: This section can be ignored if `EnableUI` -parameter is disabled.
+Please note: This section can be ignored if `EnableUI` -parameter is disabled (this is the default value).
 
-User Interface is provided as a CloudFormation Nested Stack, which means that the UI components are separated to its own template and are deployed to separate CloudFormation Stack. All Nested Stack operations should be initiated from the root stack.
+User Interface is provided as part of the `metaflow-cfn-template.yml` template and doesn't require any additional
+configuration besides enabling the `EnableUI` -parameter. You can follow the [AWS CloudFormation Deployment](https://admin-docs.metaflow.org/metaflow-on-aws/deployment-guide/aws-cloudformation-deployment#steps-for-aws-cloudformation-deployment) instructions.
 
-While working with Nested Stacks, templates need to be packaged before deployment. For this AWS Command Line Interface can be used. Packaging step is only required when deploying Metaflow User Interface Nested Stack.
+Once deployed the Cloudformation Stack will provide two outputs:
+- `UIServiceUrl` - Application Load Balancer endpoint
+- `UIServiceCloudfrontUrl` - Cloudfront distribution (using ALB) endpoint with HTTPS enabled (preferred)
 
-Prerequisites for User Interface enabled Nested Stack deployment:
-
-* **[Netflix/metaflow-tools repository](https://github.com/Netflix/metaflow-tools)** - Locally available for packaging templates.
-* **AWS Command Line Interface** - Installed and configured.
-* **S3 bucket to upload CloudFormation templates to** - Nested Stacks require templates referred by absolute S3 paths.
-
-Packaging step is required in order to deploy templates with Nested Stacks:
-
-```bash
-$ aws cloudformation package \
-    --s3-bucket manually-created-s3-bucket \
-    --template-file metaflow-cfn-template.yml \
-    --output-template-file metaflow-cfn-template.yml.package
-```
-
-Above package command does the following:
-
-1. Nested Stack template is uploaded to S3 bucket
-    1. Defined by `--s3-bucket` parameter (this bucket should already exist and needs to be created manually)
-2. Root template is transformed so that all local paths are converted to absolute S3 paths
-    1. Input file is defined by `--template-file` parameter
-3. Transformed and packaged root template is placed in local file system defined by `--output-template-file` parameter
-
-Once this package step is completed, you can follow the [AWS CloudFormation Deployment](https://admin-docs.metaflow.org/metaflow-on-aws/deployment-guide/aws-cloudformation-deployment#steps-for-aws-cloudformation-deployment). Please note: instead of using the non-packaged template, you should use the packaged template `metaflow-cfn-template.yml.package` you just created.
-
-Read more about Cloudformation and working with nested stacks [here](https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/using-cfn-nested-stacks.html).
+Please note: Metaflow User Interface doesn't provide any authentication by default.
diff --git a/aws/cloudformation/metaflow-cfn-template.yml b/aws/cloudformation/metaflow-cfn-template.yml
@@ -1,5 +1,5 @@
 AWSTemplateFormatVersion: '2010-09-09'
-Description: Stack for complete deployment of Metaflow (last-updated-date 02/09/2021)
+Description: Stack for complete deployment of Metaflow (last-updated-date 10/26/2021)
 
 Parameters:
   SagemakerInstance:
@@ -101,6 +101,22 @@ Mappings:
       value: 1
     Role:
       value: ""
+
+  ServiceInfoUI:
+    ServiceName:
+      value: 'metaflow-ui-service'
+    ImageUrl:
+      #value: 'netflixoss/metaflow_metadata_service'
+      value: '847940006482.dkr.ecr.us-west-2.amazonaws.com/metaflow-service:cloudformation' # TODO: Public Docker Hub image here
+    ContainerPort:
+      value: 8083
+    ContainerCpu:
+      value: 512
+    ContainerMemory:
+      value: 1024
+    DesiredCount:
+      value: 1
+
 Conditions:
   EnableAuth: !Equals [ !Ref APIBasicAuth, 'true']
   EnableRole: !Equals [ !Ref CustomRole, 'true']
@@ -1364,38 +1380,235 @@ Resources:
         AttributeName: ttl
         Enabled: true
 
-# Metaflow UI Nested stack
+# --- Metaflow UI resources begin ---
 
-  MetaflowUI:
-    Condition: 'EnableUI'
+  # UI: Ingress rules
+
+  IngressRuleUIService:
+    Type: AWS::EC2::SecurityGroupIngress
+    Condition: EnableUI
+    Properties:
+      Description: "Allow API Calls Internally (UI service)"
+      GroupId: !Ref FargateSecurityGroup
+      IpProtocol: tcp
+      FromPort: !FindInMap ["ServiceInfoUI", "ContainerPort", "value"]
+      ToPort: !FindInMap ["ServiceInfoUI", "ContainerPort", "value"]
+      CidrIp: !GetAtt 'VPC.CidrBlock'
+
+  # UI: Load balancer
+
+  ALBUISecurityGroup:
+    Type: AWS::EC2::SecurityGroup
+    Condition: EnableUI
+    Properties:
+      GroupDescription: "Access to the public facing load balancer"
+      VpcId: !Ref VPC
+      SecurityGroupIngress:
+        # Allow access to ALB from anywhere on the internet
+        - CidrIp: 0.0.0.0/0
+          IpProtocol: -1
+  ALBUI:
+    Type: AWS::ElasticLoadBalancingV2::LoadBalancer
+    Condition: EnableUI
+    Properties:
+      Scheme: internet-facing
+      Type: application
+      Subnets:
+        - !Ref Subnet1
+        - !Ref Subnet2
+      SecurityGroups: [!Ref ALBUISecurityGroup]
+  ALBUIListener:
+    Type: AWS::ElasticLoadBalancingV2::Listener
+    Condition: EnableUI
     DependsOn:
-      - VPC
-      - Subnet1
-      - Subnet2
-      - FargateSecurityGroup
-      - ECSCluster
-      - ECSTaskExecutionRole
-      - MetadataSvcECSTaskRole
-      - RDSMasterInstance
-      - MyRDSSecret
-      - NLB
-    Type: AWS::CloudFormation::Stack
-    Properties:
-      TemplateURL: ui/metaflow-ui-cfn-template.yml
-      Parameters:
-        VPCId: !Ref VPC
-        VPCCidrBlock: !GetAtt 'VPC.CidrBlock'
-        SubnetIds: !Join [',', [ !Ref Subnet1, !Ref Subnet2 ]]
-        SecurityGroup: !Ref FargateSecurityGroup
-        ECSClusterId: !Ref ECSCluster
-        ECSTaskExecutionRoleId: !Ref ECSTaskExecutionRole
-        MetadataSvcECSTaskRoleArn: !GetAtt 'MetadataSvcECSTaskRole.Arn'
-        DatabaseHost: !GetAtt 'RDSMasterInstance.Endpoint.Address'
-        DatabasePort: "5432"
-        DatabaseUser: "master"
-        DatabasePassword: !Join ['', ['{{resolve:secretsmanager:', !Ref MyRDSSecret, ':SecretString:password}}' ]]
-        DatabaseName: "metaflow"
-        MetaflowServiceUrl: !Sub "http://${NLB.DNSName}/"
+      - ALBUI
+    Properties:
+      DefaultActions:
+        - TargetGroupArn: !Ref ALBUITargetGroupUIService
+          Type: "forward"
+      LoadBalancerArn: !Ref ALBUI
+      Port: 80
+      Protocol: HTTP
+  ALBUITargetGroupUIService:
+    Type: AWS::ElasticLoadBalancingV2::TargetGroup
+    Condition: EnableUI
+    Properties:
+      HealthCheckIntervalSeconds: 6
+      HealthCheckPath: /ping
+      HealthCheckProtocol: HTTP
+      HealthCheckTimeoutSeconds: 5
+      HealthyThresholdCount: 2
+      TargetType: ip
+      Port: !FindInMap ["ServiceInfoUI", "ContainerPort", "value"]
+      Protocol: HTTP
+      UnhealthyThresholdCount: 2
+      VpcId: !Ref VPC
+
+  # UI: Coudfront distribution
+
+  CloudFrontDistributionUI:
+    Type: AWS::CloudFront::Distribution
+    Condition: EnableUI
+    DependsOn:
+      - ALBUI
+    Properties:
+      DistributionConfig:
+        Origins:
+          - DomainName: !GetAtt "ALBUI.DNSName"
+            Id: !Ref ALBUI
+            CustomOriginConfig:
+              HTTPPort: 80
+              OriginProtocolPolicy: http-only
+              OriginKeepaliveTimeout: 60
+              OriginReadTimeout: 30
+              OriginSSLProtocols:
+                - TLSv1
+                - TLSv1.1
+                - TLSv1.2
+                - SSLv3
+        Enabled: true
+        HttpVersion: http2
+        DefaultCacheBehavior:
+          AllowedMethods:
+            - GET
+            - HEAD
+            - DELETE
+            - OPTIONS
+            - PATCH
+            - POST
+            - PUT
+          Compress: true
+          DefaultTTL: 0
+          MinTTL: 0
+          MaxTTL: 0
+          SmoothStreaming: false
+          TargetOriginId: !Ref ALBUI
+          ForwardedValues:
+            QueryString: true
+            Cookies:
+              Forward: none
+          ViewerProtocolPolicy: redirect-to-https
+        CacheBehaviors:
+          - AllowedMethods:
+              - GET
+              - HEAD
+              - DELETE
+              - OPTIONS
+              - PATCH
+              - POST
+              - PUT
+            Compress: false
+            PathPattern: /api/*
+            DefaultTTL: 0
+            MinTTL: 0
+            MaxTTL: 0
+            SmoothStreaming: false
+            TargetOriginId: !Ref ALBUI
+            ForwardedValues:
+              QueryString: true
+              Cookies:
+                Forward: none
+            ViewerProtocolPolicy: redirect-to-https
+        PriceClass: PriceClass_All
+        IPV6Enabled: true
+
+  # UI: ECS Cluster task definition & service
+
+  TaskDefinitionUIService:
+    Type: AWS::ECS::TaskDefinition
+    Condition: EnableUI
+    Properties:
+      Family: !FindInMap ["ServiceInfoUI", "ServiceName", "value"]
+      Cpu: !FindInMap ["ServiceInfoUI", "ContainerCpu", "value"]
+      Memory: !FindInMap ["ServiceInfoUI", "ContainerMemory", "value"]
+      NetworkMode: awsvpc
+      RequiresCompatibilities:
+        - FARGATE
+      ExecutionRoleArn: !Ref 'ECSTaskExecutionRole'
+      TaskRoleArn: !GetAtt 'MetadataSvcECSTaskRole.Arn'
+      ContainerDefinitions:
+        - Name: !FindInMap ["ServiceInfoUI", "ServiceName", "value"]
+          Environment:
+            - Name: "MF_METADATA_DB_HOST"
+              Value: !GetAtt 'RDSMasterInstance.Endpoint.Address'
+            - Name: "MF_METADATA_DB_PORT"
+              Value: "5432"
+            - Name: "MF_METADATA_DB_USER"
+              Value: !Join ['', ['{{resolve:secretsmanager:', !Ref MyRDSSecret, ':SecretString:username}}' ]]
+            - Name: "MF_METADATA_DB_PSWD"
+              Value: !Join ['', ['{{resolve:secretsmanager:', !Ref MyRDSSecret, ':SecretString:password}}' ]]
+            - Name: "MF_METADATA_DB_NAME"
+              Value: "metaflow"
+            - Name: "UI_ENABLED"
+              Value: "1"
+          Cpu: !FindInMap ["ServiceInfoUI", "ContainerCpu", "value"]
+          Memory: !FindInMap ["ServiceInfoUI", "ContainerMemory", "value"]
+          Image: !FindInMap ["ServiceInfoUI", "ImageUrl", "value"]
+          Command:
+            [
+              "/opt/latest/bin/python3",
+              "-m",
+              "services.ui_backend_service.ui_server",
+            ]
+          PortMappings:
+            - ContainerPort:
+                !FindInMap ["ServiceInfoUI", "ContainerPort", "value"]
+          LogConfiguration:
+            LogDriver: awslogs
+            Options:
+              awslogs-group:
+                !Join [
+                  "",
+                  [
+                    "/ecs/",
+                    !Ref "AWS::StackName",
+                    "-",
+                    !FindInMap ["ServiceInfoUI", "ServiceName", "value"],
+                  ],
+                ]
+              awslogs-region: !Ref "AWS::Region"
+              awslogs-stream-prefix: "ecs"
+  ServiceLogGroupUIService:
+    Type: AWS::Logs::LogGroup
+    Condition: EnableUI
+    Properties:
+      LogGroupName:
+        !Join [
+          "",
+          [
+            "/ecs/",
+            !Ref "AWS::StackName",
+            "-",
+            !FindInMap ["ServiceInfoUI", "ServiceName", "value"],
+          ],
+        ]
+  ECSFargateServiceUIService:
+    Type: AWS::ECS::Service
+    Condition: EnableUI
+    DependsOn: ALBUIListener
+    Properties:
+      ServiceName: !FindInMap ["ServiceInfoUI", "ServiceName", "value"]
+      Cluster: !Ref 'ECSCluster'
+      LaunchType: FARGATE
+      DeploymentConfiguration:
+        MaximumPercent: 200
+        MinimumHealthyPercent: 75
+      DesiredCount: !FindInMap ["ServiceInfoUI", "DesiredCount", "value"]
+      NetworkConfiguration:
+        AwsvpcConfiguration:
+          AssignPublicIp: ENABLED
+          SecurityGroups:
+            - !Ref FargateSecurityGroup
+          Subnets:
+            - !Ref Subnet1
+            - !Ref Subnet2
+      TaskDefinition: !Ref TaskDefinitionUIService
+      LoadBalancers:
+        - ContainerName: !FindInMap ["ServiceInfoUI", "ServiceName", "value"]
+          ContainerPort: !FindInMap ["ServiceInfoUI", "ContainerPort", "value"]
+          TargetGroupArn: !Ref ALBUITargetGroupUIService
+
+# --- Metaflow UI resources end ---
 
 Outputs:
   MetaflowDataStoreS3Url:
@@ -1440,3 +1653,14 @@ Outputs:
   MigrationFunctionName:
     Description: "Name of DB Migration Function"
     Value: !Ref ExecuteDBMigration
+  
+  # Metaflow UI outputs
+
+  UIServiceUrl:
+    Description: "ALB URL for Metadata UI (Open to Public Access)"
+    Value: !Sub "http://${ALBUI.DNSName}/"
+    Condition: EnableUI
+  UIServiceCloudfrontUrl:
+    Description: "Cloudfront URL for Metadata UI (Open to Public Access)"
+    Value: !Sub "https://${CloudFrontDistributionUI.DomainName}/"
+    Condition: EnableUI
diff --git a/aws/cloudformation/ui/metaflow-ui-cfn-template.yml b/aws/cloudformation/ui/metaflow-ui-cfn-template.yml
