From e4f7919ca16dceba4bc4144b6fdd53f501a9775a Mon Sep 17 00:00:00 2001 From: Nidhi Work Date: Wed, 3 Jul 2024 17:18:33 +0100 Subject: [PATCH 1/2] feat(ci): test CF templates when published --- .../npm-publish-all-packages-canary.yml | 7 +- .../workflows/npm-publish-all-packages.yml | 7 +- .github/workflows/s3-publish-cf-templates.yml | 174 +++++++++++++++++- .../aws/iam-cf-templates/gh-oidc-lambda.yml | 2 +- 4 files changed, 182 insertions(+), 8 deletions(-) diff --git a/.github/workflows/npm-publish-all-packages-canary.yml b/.github/workflows/npm-publish-all-packages-canary.yml index 63a4595cf4..63c19b793b 100644 --- a/.github/workflows/npm-publish-all-packages-canary.yml +++ b/.github/workflows/npm-publish-all-packages-canary.yml @@ -93,7 +93,7 @@ jobs: DD_TESTS_APP_KEY: ${{ secrets.DD_TESTS_APP_KEY }} AWS_TEST_EXECUTION_ROLE_ARN_TEST5: ${{ secrets.AWS_TEST_EXECUTION_ROLE_ARN_TEST5 }} - publish-cloudformation-templates-canary-to-s3: + run-cloudformation-iam-setup-tests: uses: ./.github/workflows/s3-publish-cf-templates.yml needs: run-distributed-tests with: @@ -103,3 +103,8 @@ jobs: id-token: write secrets: AWS_ASSET_UPLOAD_ROLE_ARN: ${{ secrets.AWS_ASSET_UPLOAD_ROLE_ARN }} + ARTILLERY_CLOUD_ENDPOINT_TEST: ${{ secrets.ARTILLERY_CLOUD_ENDPOINT_TEST }} + ARTILLERY_CLOUD_API_KEY_TEST: ${{ secrets.ARTILLERY_CLOUD_API_KEY_TEST }} + AWS_ACCOUNT_ID_FRESH_SETUP_TESTS: ${{ secrets.AWS_ACCOUNT_ID_FRESH_SETUP_TESTS }} + DD_TESTS_API_KEY: ${{ secrets.DD_TESTS_API_KEY }} + DD_TESTS_APP_KEY: ${{ secrets.DD_TESTS_APP_KEY }} diff --git a/.github/workflows/npm-publish-all-packages.yml b/.github/workflows/npm-publish-all-packages.yml index f18476d50c..a93171aa13 100644 --- a/.github/workflows/npm-publish-all-packages.yml +++ b/.github/workflows/npm-publish-all-packages.yml @@ -109,10 +109,13 @@ jobs: publish-cloudformation-templates-to-s3: uses: ./.github/workflows/s3-publish-cf-templates.yml needs: run-distributed-tests - with: - canary: true permissions: contents: read id-token: write secrets: AWS_ASSET_UPLOAD_ROLE_ARN: ${{ secrets.AWS_ASSET_UPLOAD_ROLE_ARN }} + ARTILLERY_CLOUD_ENDPOINT_TEST: ${{ secrets.ARTILLERY_CLOUD_ENDPOINT_TEST }} + ARTILLERY_CLOUD_API_KEY_TEST: ${{ secrets.ARTILLERY_CLOUD_API_KEY_TEST }} + AWS_ACCOUNT_ID_FRESH_SETUP_TESTS: ${{ secrets.AWS_ACCOUNT_ID_FRESH_SETUP_TESTS }} + DD_TESTS_API_KEY: ${{ secrets.DD_TESTS_API_KEY }} + DD_TESTS_APP_KEY: ${{ secrets.DD_TESTS_APP_KEY }} diff --git a/.github/workflows/s3-publish-cf-templates.yml b/.github/workflows/s3-publish-cf-templates.yml index 950e2b9ed0..b8201801d1 100644 --- a/.github/workflows/s3-publish-cf-templates.yml +++ b/.github/workflows/s3-publish-cf-templates.yml @@ -11,6 +11,21 @@ on: AWS_ASSET_UPLOAD_ROLE_ARN: description: 'ARN of the IAM role to assume to upload assets to S3' required: true + AWS_ACCOUNT_ID_FRESH_SETUP_TESTS: + description: 'AWS Account ID to use for setting up IAM permissions and running tests' + required: true + ARTILLERY_CLOUD_ENDPOINT_TEST: + description: 'Artillery Cloud endpoint for running tests' + required: true + ARTILLERY_CLOUD_API_KEY_TEST: + description: 'Artillery Cloud API Key for running tests' + required: true + DD_TESTS_API_KEY: + description: 'Datadog API Key for running tests' + required: true + DD_TESTS_APP_KEY: + description: 'Datadog App Key for running tests' + required: true workflow_dispatch: inputs: @@ -22,9 +37,10 @@ on: env: CF_LAMBDA_TEMPLATE: ${{ inputs.canary && 'aws-iam-lambda-cf-template-canary.yml' || 'aws-iam-lambda-cf-template.yml' }} CF_FARGATE_TEMPLATE: ${{ inputs.canary && 'aws-iam-fargate-cf-template-canary.yml' || 'aws-iam-fargate-cf-template.yml' }} - GH_OIDC_LAMBDA_TEMPLATE: ${{ inputs.canary && 'gh-oidc-lambda-canary.yml' || 'gh-oidc-lambda.yml' }} - GH_OIDC_FARGATE_TEMPLATE: ${{ inputs.canary && 'gh-oidc-fargate-canary.yml' || 'gh-oidc-fargate.yml' }} + GH_OIDC_TEMPLATE: ${{ inputs.canary && 'github-oidc-canary.yml' || 'github-oidc.yml' }} + jobs: + # Publish templates to AWS S3 put-cloudformation-templates: runs-on: ubuntu-latest @@ -50,5 +66,155 @@ jobs: run: | aws s3 cp --acl public-read ./packages/artillery/lib/platform/aws/iam-cf-templates/aws-iam-fargate-cf-template.yml s3://artilleryio-cf-templates/${{ env.CF_FARGATE_TEMPLATE }} aws s3 cp --acl public-read ./packages/artillery/lib/platform/aws/iam-cf-templates/aws-iam-lambda-cf-template.yml s3://artilleryio-cf-templates/${{ env.CF_LAMBDA_TEMPLATE }} - aws s3 cp --acl public-read ./packages/artillery/lib/platform/aws/iam-cf-templates/gh-oidc-lambda.yml s3://artilleryio-cf-templates/${{ env.GH_OIDC_LAMBDA_TEMPLATE }} - aws s3 cp --acl public-read ./packages/artillery/lib/platform/aws/iam-cf-templates/gh-oidc-fargate.yml s3://artilleryio-cf-templates/${{ env.GH_OIDC_FARGATE_TEMPLATE }} \ No newline at end of file + aws s3 cp --acl public-read ./packages/artillery/lib/platform/aws/iam-cf-templates/github-oidc.yml s3://artilleryio-cf-templates/${{ env.GH_OIDC_TEMPLATE }} + + # This job is used to test that the IAM role created from the `github-oidc.yml` CF template has the correct permissions to be able to run Artillery tests on Lambda and Fargate and that the OIDC is set correctly. + set-up-and-run-artillery-test-on-aws: + runs-on: ubuntu-latest + + permissions: + id-token: write + contents: read + needs: put-cloudformation-templates + strategy: + matrix: + service: [lambda, fargate] + include: + - service: lambda + test-file: 'test/cloud-e2e/lambda/lambda-smoke.test.js' + - service: fargate + test-file: 'test/cloud-e2e/fargate/dd-adot.test.js' + max-parallel: 1 + + steps: + - name: Checkout code + uses: actions/checkout@v2 + + - name: Configure AWS Credentials + uses: aws-actions/configure-aws-credentials@v2 + env: + SHOW_STACK_TRACE: true + with: + aws-region: us-east-1 + role-to-assume: arn:aws:iam::${{ secrets.AWS_ACCOUNT_ID_FRESH_SETUP_TESTS }}:role/ResetAccountForRunningTests + role-session-name: OIDCSession + mask-aws-account-id: true + + - name: Install aws-nuke + run: | + curl -sL https://github.com/rebuy-de/aws-nuke/releases/download/v2.25.0/aws-nuke-v2.25.0-linux-amd64.tar.gz | tar -xz + sudo install aws-nuke-v2.25.0-linux-amd64 /usr/local/bin/aws-nuke + + - name: Run aws-nuke + run: | + cat < aws-nuke-config.yml + regions: + - "global" + - "us-east-2" + - "us-east-1" + - "us-west-1" + - "us-west-2" + - "ap-south-1" + - "ap-northeast-3" + - "ap-northeast-2" + - "ap-southeast-1" + - "ap-southeast-2" + - "ap-northeast-1" + - "ca-central-1" + - "eu-central-1" + - "eu-west-1" + - "eu-west-2" + - "eu-west-3" + - "eu-north-1" + - "sa-east-1" + + account-blocklist: + - 111111111111 + + resource-types: + targets: + - IAMRole + - IAMPolicy + - IAMRolePolicyAttachment + - IAMOpenIDConnectProvider + - S3Bucket + - S3Object + - LambdaFunction + - ECSTaskDefinition + - ECSCluster + - ECSContainerInstance + - CloudFormationStack + - SQSQueue + - CloudWatchLogGroup + - CloudWatchLogStream + - SSMParameter + + accounts: + "${{ secrets.AWS_ACCOUNT_ID_FRESH_SETUP_TESTS }}": + filters: + IAMRole: + - property: Name + type: "glob" + value: "AWS*" + - property: Name + type: "glob" + value: "Organization*" + - property: Name + value: "ResetAccountForRunningTests" + + IAMRolePolicyAttachment: + - property: RoleName + type: "glob" + value: "AWS*" + - property: RoleName + value: "ResetAccountForRunningTests" + - property: RoleName + type: "glob" + value: "Organization*" + + IAMOpenIDConnectProvider: + - property: Arn + type: "contains" + value: "oidc-provider/token.actions.githubusercontent.com" + invert: true + EOF + + aws-nuke --config aws-nuke-config.yml --force --force-sleep 4 --no-dry-run + + - name: Fetch CloudFormation Template + run: | + aws s3 cp s3://artilleryio-cf-templates/${{ env.GH_OIDC_TEMPLATE }} template.yml + - name: Deploy CloudFormation Template + run: | + aws cloudformation create-stack \ + --stack-name iam-distributed-testing-setup \ + --template-body file://template.yml \ + --parameters ParameterKey=GitHubRepository,ParameterValue="artilleryio/artillery" \ + --capabilities CAPABILITY_NAMED_IAM + aws cloudformation wait stack-create-complete --stack-name iam-distributed-testing-setup + + - name: Configure AWS Credentials + uses: aws-actions/configure-aws-credentials@v2 + env: + SHOW_STACK_TRACE: true + with: + aws-region: us-east-1 + role-to-assume: arn:aws:iam::${{ secrets.AWS_ACCOUNT_ID_FRESH_SETUP_TESTS }}:role/ArtilleryGitHubOIDCRole + role-session-name: OIDCSession + mask-aws-account-id: true + - name: Use Node.js 18.x + uses: actions/setup-node@v2 + with: + node-version: 18.x + - run: .github/workflows/scripts/npm-command-retry.sh install + - run: npm run build + - name: Run Artillery Test + env: + ARTILLERY_CLOUD_ENDPOINT: ${{ secrets.ARTILLERY_CLOUD_ENDPOINT_TEST }} + ARTILLERY_CLOUD_API_KEY: ${{ secrets.ARTILLERY_CLOUD_API_KEY_TEST }} + FORCE_COLOR: 1 + DD_TESTS_API_KEY: ${{ secrets.DD_TESTS_API_KEY }} + DD_TESTS_APP_KEY: ${{ secrets.DD_TESTS_APP_KEY }} + run: | + npm run test:aws:ci --workspace artillery -- --files ${{ matrix.test-file }} + \ No newline at end of file diff --git a/packages/artillery/lib/platform/aws/iam-cf-templates/gh-oidc-lambda.yml b/packages/artillery/lib/platform/aws/iam-cf-templates/gh-oidc-lambda.yml index 41ad2710a2..ce063b3e3a 100644 --- a/packages/artillery/lib/platform/aws/iam-cf-templates/gh-oidc-lambda.yml +++ b/packages/artillery/lib/platform/aws/iam-cf-templates/gh-oidc-lambda.yml @@ -74,7 +74,7 @@ Resources: Fn::If: - CreateOIDCProvider - !Ref GitHubOIDCProvider - - !Ref GitHubOIDCProviderArn + - !Sub "arn:aws:iam::${AWS::AccountId}:oidc-provider/token.actions.githubusercontent.com" Action: "sts:AssumeRoleWithWebIdentity" Condition: { StringEquals: From 2c68262bfa755f2c71e7226739a66566f510d1a4 Mon Sep 17 00:00:00 2001 From: Nidhi Work Date: Tue, 9 Jul 2024 09:36:52 +0100 Subject: [PATCH 2/2] refactor: set Lambda and Fargate policy in same CF template for GH OIDC --- .../aws/iam-cf-templates/gh-oidc-lambda.yml | 153 ------------------ .../{gh-oidc-fargate.yml => github-oidc.yml} | 73 ++++++++- 2 files changed, 65 insertions(+), 161 deletions(-) delete mode 100644 packages/artillery/lib/platform/aws/iam-cf-templates/gh-oidc-lambda.yml rename packages/artillery/lib/platform/aws/iam-cf-templates/{gh-oidc-fargate.yml => github-oidc.yml} (75%) diff --git a/packages/artillery/lib/platform/aws/iam-cf-templates/gh-oidc-lambda.yml b/packages/artillery/lib/platform/aws/iam-cf-templates/gh-oidc-lambda.yml deleted file mode 100644 index ce063b3e3a..0000000000 --- a/packages/artillery/lib/platform/aws/iam-cf-templates/gh-oidc-lambda.yml +++ /dev/null @@ -1,153 +0,0 @@ -AWSTemplateFormatVersion: '2010-09-09' -Description: Creates an ArtilleryGitHubOIDCForLambdaRole IAM role with permissions needed to run Artillery Lambda tests from a specified GitHub repository. An OIDC identity provider for Github will also be created if it is not already present in the account. - - -Metadata: - AWS::CloudFormation::Interface: - ParameterGroups: - - Label: - default: "GitHub" - Parameters: - - GitHubRepository - - GitHubBranch - - Label: - default: "AWS IAM" - Parameters: - - GitHubOIDCProviderExists - - ParameterLabels: - GitHubRepository: - default: "GitHub repository" - GitHubBranch: - default: "GitHub branch" - GitHubOIDCProviderExists: - default: "GitHub OIDC identity provider already created for the account?" - -Parameters: - GitHubRepository: - Type: String - Default: "" - Description: The GitHub repository (orgname/reponame) to be allowed to assume the created IAM role using OIDC (e.g. "artilleryio/artillery"). - - GitHubBranch: - Type: String - Default: "*" - Description: (Optional) Use when you want to allow only a specific branch within the specified Github repository to assume this IAM role using OIDC (e.g. "main"). If not set, defaults to "*" (all branches). - - GitHubOIDCProviderExists: - Type: String - Default: 'No' - AllowedValues: - - 'Yes' - - 'No' - Description: This will let CloudFormation know whether it needs to create the provider. (If it exists, can be found at Services -> IAM -> Identity providers as 'token.actions.githubusercontent.com'). - -Conditions: - IsGHRepoSet: - !Not [!Equals [!Ref GitHubRepository, ""]] - - CreateOIDCProvider: - !Equals [!Ref GitHubOIDCProviderExists, "No"] - -Resources: - GitHubOIDCProvider: - Type: AWS::IAM::OIDCProvider - Condition: CreateOIDCProvider - Properties: - Url: "https://token.actions.githubusercontent.com" - ClientIdList: - - "sts.amazonaws.com" - ThumbprintList: - - "6938fd4d98bab03faadb97b34396831e3780ee11" - - - ArtilleryGitHubOIDCForLambdaRole: - Type: "AWS::IAM::Role" - Properties: - RoleName: "ArtilleryGitHubOIDCForLambdaRole" - AssumeRolePolicyDocument: - Version: "2012-10-17" - Statement: - - Effect: "Allow" - Principal: - Federated: - Fn::If: - - CreateOIDCProvider - - !Ref GitHubOIDCProvider - - !Sub "arn:aws:iam::${AWS::AccountId}:oidc-provider/token.actions.githubusercontent.com" - Action: "sts:AssumeRoleWithWebIdentity" - Condition: { - StringEquals: - { - "token.actions.githubusercontent.com:aud": "sts.amazonaws.com" - }, - StringLike: - { - "token.actions.githubusercontent.com:sub": !Sub "repo:${GitHubRepository}:${GitHubBranch}" - } - } - Path: "/" - Policies: - - PolicyName: ArtilleryDistributedTestingLambdaPolicy - PolicyDocument: - Version: "2012-10-17" - Statement: - - Sid: CreateOrGetLambdaRole - Effect: Allow - Action: - - iam:CreateRole - - iam:GetRole - - iam:PassRole - - iam:AttachRolePolicy - Resource: !Sub "arn:aws:iam::${AWS::AccountId}:role/artilleryio-default-lambda-role-*" - - Sid: CreateLambdaPolicy - Effect: Allow - Action: - - iam:CreatePolicy - Resource: !Sub "arn:aws:iam::${AWS::AccountId}:policy/artilleryio-lambda-policy-*" - - Sid: SQSPermissions - Effect: Allow - Action: - - sqs:* - Resource: !Sub "arn:aws:sqs:*:${AWS::AccountId}:artilleryio*" - - Sid: SQSListQueues - Effect: Allow - Action: - - sqs:ListQueues - Resource: "*" - - Sid: LambdaPermissions - Effect: Allow - Action: - - lambda:InvokeFunction - - lambda:CreateFunction - - lambda:DeleteFunction - - lambda:GetFunctionConfiguration - Resource: !Sub "arn:aws:lambda:*:${AWS::AccountId}:function:artilleryio-*" - - Sid: EcrPullImagePermissions - Effect: Allow - Action: - - ecr:GetDownloadUrlForLayer - - ecr:BatchGetImage - Resource: "arn:aws:ecr:*:248481025674:repository/artillery-worker" - - Sid: S3Permissions - Effect: Allow - Action: - - s3:CreateBucket - - s3:DeleteObject - - s3:GetObject - - s3:PutObject - - s3:ListBucket - - s3:GetLifecycleConfiguration - - s3:PutLifecycleConfiguration - Resource: - - !Sub "arn:aws:s3:::artilleryio-test-data-*" - - !Sub "arn:aws:s3:::artilleryio-test-data-*/*" - -Outputs: - RoleArn: - Description: ARN of the IAM Role for Artillery.io Lambda functions - Value: !GetAtt ArtilleryGitHubOIDCForLambdaRole.Arn - OIDCProviderArn: - Condition: CreateOIDCProvider - Description: "ARN of the newly created OIDC provider" - Value: !Ref GitHubOIDCProvider \ No newline at end of file diff --git a/packages/artillery/lib/platform/aws/iam-cf-templates/gh-oidc-fargate.yml b/packages/artillery/lib/platform/aws/iam-cf-templates/github-oidc.yml similarity index 75% rename from packages/artillery/lib/platform/aws/iam-cf-templates/gh-oidc-fargate.yml rename to packages/artillery/lib/platform/aws/iam-cf-templates/github-oidc.yml index d94c032cfd..d5dd3440be 100644 --- a/packages/artillery/lib/platform/aws/iam-cf-templates/gh-oidc-fargate.yml +++ b/packages/artillery/lib/platform/aws/iam-cf-templates/github-oidc.yml @@ -1,5 +1,5 @@ AWSTemplateFormatVersion: '2010-09-09' -Description: Creates an ArtilleryGitHubOIDCForFargateRole IAM role with permissions needed to run Artillery Fargate tests from a specified GitHub repository. An OIDC identity provider for Github will also be created if it is not already present in the account. +Description: Sets up IAM resources needed to trigger Artillery distributed tests (on AWS Fargate/Lambda) from a specified GitHub repository. Uses OpenID Connect (OIDC) to authenticate requests from GitHub. Metadata: AWS::CloudFormation::Interface: ParameterGroups: @@ -19,13 +19,13 @@ Metadata: GitHubBranch: default: "GitHub branch" GitHubOIDCProviderExists: - default: "GitHub OIDC identity provider already created for the account?" + default: "Is GitHub OIDC identity provider already created for the account?" Parameters: GitHubRepository: Type: String Default: "" - Description: The GitHub repository (orgname/reponame) to be allowed to assume the created IAM role using OIDC (e.g. "artilleryio/artillery"). + Description: The GitHub repository ("GitHubOrganizationOrUser/GitHubRepository") to be allowed to assume the created IAM role using OIDC (e.g. "artilleryio/artillery"). GitHubBranch: Type: String @@ -38,7 +38,8 @@ Parameters: AllowedValues: - 'Yes' - 'No' - Description: This will let CloudFormation know whether it needs to create the provider. (If it exists, can be found at Services -> IAM -> Identity providers as 'token.actions.githubusercontent.com'). + Description: This will let CloudFormation know whether it needs to create the OIDC identity provider for GitHub. (If it exists, can be found at IAM > Identity providers). + Conditions: IsGHRepoSet: @@ -56,12 +57,12 @@ Resources: ClientIdList: - "sts.amazonaws.com" ThumbprintList: - - "6938fd4d98bab03faadb97b34396831e3780ee11" + - "6938fd4d98bab03faadb97b34396831e3780aea1" - ArtilleryGitHubOIDCForFargateRole: + ArtilleryGitHubOIDCRole: Type: "AWS::IAM::Role" Properties: - RoleName: "ArtilleryGitHubOIDCForFargateRole" + RoleName: "ArtilleryGitHubOIDCRole" AssumeRolePolicyDocument: Version: "2012-10-17" Statement: @@ -228,12 +229,68 @@ Resources: - "ec2:DescribeSubnets" Resource: "*" + - PolicyName: ArtilleryDistributedTestingLambdaPolicy + PolicyDocument: + Version: "2012-10-17" + Statement: + - Sid: CreateOrGetLambdaRole + Effect: Allow + Action: + - iam:CreateRole + - iam:GetRole + - iam:PassRole + - iam:AttachRolePolicy + Resource: !Sub "arn:aws:iam::${AWS::AccountId}:role/artilleryio-default-lambda-role-*" + - Sid: CreateLambdaPolicy + Effect: Allow + Action: + - iam:CreatePolicy + Resource: !Sub "arn:aws:iam::${AWS::AccountId}:policy/artilleryio-lambda-policy-*" + - Sid: SQSPermissions + Effect: Allow + Action: + - sqs:* + Resource: !Sub "arn:aws:sqs:*:${AWS::AccountId}:artilleryio*" + - Sid: SQSListQueues + Effect: Allow + Action: + - sqs:ListQueues + Resource: "*" + - Sid: LambdaPermissions + Effect: Allow + Action: + - lambda:InvokeFunction + - lambda:CreateFunction + - lambda:DeleteFunction + - lambda:GetFunctionConfiguration + Resource: !Sub "arn:aws:lambda:*:${AWS::AccountId}:function:artilleryio-*" + - Sid: EcrPullImagePermissions + Effect: Allow + Action: + - ecr:GetDownloadUrlForLayer + - ecr:BatchGetImage + Resource: "arn:aws:ecr:*:248481025674:repository/artillery-worker" + - Sid: S3Permissions + Effect: Allow + Action: + - s3:CreateBucket + - s3:DeleteObject + - s3:GetObject + - s3:PutObject + - s3:ListBucket + - s3:GetLifecycleConfiguration + - s3:PutLifecycleConfiguration + Resource: + - !Sub "arn:aws:s3:::artilleryio-test-data-*" + - !Sub "arn:aws:s3:::artilleryio-test-data-*/*" + + Outputs: RoleArn: Description: "ARN of the created IAM Role" Value: Fn::GetAtt: - - "ArtilleryGitHubOIDCForFargateRole" + - "ArtilleryGitHubOIDCRole" - "Arn" OIDCProviderArn: Condition: CreateOIDCProvider