diff --git a/.github/workflows/deploy.yaml b/.github/workflows/deploy.yaml index 94f6355..d59e047 100644 --- a/.github/workflows/deploy.yaml +++ b/.github/workflows/deploy.yaml @@ -20,7 +20,7 @@ jobs: deploy: name: Upload to Amazon S3 runs-on: ubuntu-latest - if: ${{ github.ref == 'refs/heads/master' || startsWith(github.ref, 'refs/tags/v') }} + if: ${{ github.ref == 'refs/heads/master' || startsWith(github.ref, 'refs/heads/test/') || startsWith(github.ref, 'refs/tags/v') }} needs: [ "validate" ] permissions: id-token: write diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 25f0aec..19df504 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,3 +1,6 @@ +ci: + autoupdate_schedule: monthly + repos: - repo: https://github.com/awslabs/git-secrets rev: b9e96b3212fa06aea65964ff0d5cda84ce935f38 @@ -6,7 +9,7 @@ repos: entry: git-secrets args: [--scan, --recursive] - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v4.5.0 + rev: v5.0.0 hooks: # On Windows, git will convert all CRLF to LF, # but only after all hooks are done executing. @@ -18,20 +21,20 @@ repos: - id: end-of-file-fixer - id: trailing-whitespace - repo: https://github.com/adrienverge/yamllint - rev: v1.32.0 + rev: v1.35.1 hooks: - id: yamllint - repo: https://github.com/aws-cloudformation/cfn-python-lint - rev: v0.83.1 + rev: v1.20.2 hooks: - id: cfn-python-lint files: .*/.*\.(json|yml|yaml)$ - repo: https://github.com/Lucas-C/pre-commit-hooks - rev: v1.5.4 + rev: v1.5.5 hooks: - id: remove-tabs - repo: https://github.com/sirosen/check-jsonschema - rev: 0.27.1 + rev: 0.30.0 hooks: - id: check-github-workflows - id: check-github-actions diff --git a/README.md b/README.md index e3d4446..b757cfb 100644 --- a/README.md +++ b/README.md @@ -12,27 +12,34 @@ This guide will help you deploy and manage your AWS ServiceCatalog using Infrast ## Deployment Merges to master will copy these templates to an S3 bucket that is used by [organizations-infra/sceptre/scipool](https://github.com/Sage-Bionetworks-IT/organizations-infra/tree/master/sceptre/scipool) to deploy stacks. -## Development -Currently there is a manual process for development. The steps are: -1. Deploy a test service catalog product template to an S3 bucket that has public access. -2. Reference that template in a template that deploys a product. Examples can be found in the individual - [Sceptre templates in org-formation-infra repo](https://github.com/Sage-Bionetworks-IT/organizations-infra/tree/master/sceptre/scipool/config/develop) - -Example: -```yaml -template: - path: "sc-product-ec2-linux-docker.j2" -stack_name: "my-sc-product-ec2-linux-docker" -parameters: - ProductName: "EC2: Linux Docker" -sceptre_user_data: - # force cloudformation to update stack by setting a random number to the latest product's description - ProvisioningArtifactParameters: | - - Description: 'Test EC2 docker' - Info: - LoadTemplateFromURL: 'https://.s3.amazonaws.com/templates/ec2/sc-ec2-linux-docker.yaml' - Name: 'v9.9.9' -``` +## Develoopment + +### Pre-Commit +As a pre-deployment step we syntatically validate our sceptre and +cloudformation yaml files with [pre-commit](https://pre-commit.com). + +Please install pre-commit, once installed the file validations will +automatically run on every commit. Alternatively you can manually +execute the validations by running `pre-commit run --all-files`. +Please install pre-commit, once installed the file validations will +automatically run on every commit. + +### Functional Testing +In order to test a change to a template, the template to test needs +to be uploaded to S3, and then a service catalog product needs to +be configured to use the test template. + +This process has been automated by running the deploy pipeline on +branches that begin with `test/`, which will upload the templates +to S3 in a directory named after the branch name. Automating the +process increases consistency and accuracy of the tests, and +reduces the impact of potential human error. + +Once a test template has been uploaded to S3, a PR can be made +to `organizations-infra` to create or update a service catalog +product that references the test template, like in [this PR](https://github.com/Sage-Bionetworks-IT/organizations-infra/pull/1109/files). +This process is outlined [in scipool documentation in `organizations-infra`](https://github.com/Sage-Bionetworks-IT/organizations-infra/tree/master/sceptre/scipool/README.md) +and existing templates can be found in the individual [sceptre templates in `organiationss-infra`](https://github.com/Sage-Bionetworks-IT/organizations-infra/tree/master/sceptre/scipool/config/develop). ### Account and Credentials While the templates in this library could be used in any AWS account, Sage Bionetworks is currently deploying to the "scipooldev" and "scipoolprod" account. diff --git a/templates/batch/sc-batch-fargate.yaml b/templates/batch/sc-batch-fargate.yaml index 5567e55..dbab565 100644 --- a/templates/batch/sc-batch-fargate.yaml +++ b/templates/batch/sc-batch-fargate.yaml @@ -206,6 +206,7 @@ Resources: config: ignore_checks: # Ignore cfn-lint errors for PyPlate. - E3002 + - E3012 Type: AWS::Batch::JobDefinition Properties: Type: container diff --git a/templates/ec2/sc-ec2-linux-docker-notebook.yaml b/templates/ec2/sc-ec2-linux-docker-notebook.yaml index 563e296..0e650cd 100644 --- a/templates/ec2/sc-ec2-linux-docker-notebook.yaml +++ b/templates/ec2/sc-ec2-linux-docker-notebook.yaml @@ -5,6 +5,7 @@ Metadata: config: ignore_checks: - E7001 + - W6001 AWS::CloudFormation::Interface: ParameterGroups: - Label: @@ -155,8 +156,6 @@ Resources: 'Fn::FindInMap': [AccountToImportParams, !Ref "AWS::AccountId", VPCId] SecurityGroupIngress: - CidrIp: "10.50.0.0/16" - FromPort: -1 - ToPort: -1 IpProtocol: "-1" InstanceProfile: @@ -274,7 +273,7 @@ Resources: files: /opt/sage/bin/make_env_vars_file.sh: source: "https://raw.githubusercontent.com/Sage-Bionetworks/service-catalog-utils/v2.0.3/linux/opt/sage/bin/make_env_vars_file.sh" - mode: "00744" + mode: "000744" owner: "root" group: "root" commands: diff --git a/templates/ec2/sc-ec2-linux-docker.yaml b/templates/ec2/sc-ec2-linux-docker.yaml index 75cbc95..5d43079 100644 --- a/templates/ec2/sc-ec2-linux-docker.yaml +++ b/templates/ec2/sc-ec2-linux-docker.yaml @@ -86,8 +86,6 @@ Resources: 'Fn::FindInMap': [AccountToImportParams, !Ref "AWS::AccountId", VPCId] SecurityGroupIngress: - CidrIp: "10.50.0.0/16" - FromPort: -1 - ToPort: -1 IpProtocol: "-1" InstanceRole: Type: AWS::IAM::Role @@ -176,7 +174,7 @@ Resources: files: /opt/sage/bin/make_env_vars_file.sh: source: "https://raw.githubusercontent.com/Sage-Bionetworks/service-catalog-utils/v2.1.0/linux/opt/sage/bin/make_env_vars_file.sh" - mode: "00744" + mode: "000744" owner: "root" group: "root" commands: diff --git a/templates/ec2/sc-ec2-windows-jumpcloud.yaml b/templates/ec2/sc-ec2-windows-jumpcloud.yaml index 63f6761..aba30c5 100644 --- a/templates/ec2/sc-ec2-windows-jumpcloud.yaml +++ b/templates/ec2/sc-ec2-windows-jumpcloud.yaml @@ -114,7 +114,7 @@ Resources: files: 'c:\\scripts\\install-chocolatey.ps1': source: "https://chocolatey.org/install.ps1" - mode: "0664" + mode: "000664" commands: 01_install_nuget: command: 'Powershell.exe Install-PackageProvider -Name NuGet -Force' @@ -133,7 +133,7 @@ Resources: files: 'c:\\scripts\\set_env_vars_file.ps1': source: "https://raw.githubusercontent.com/Sage-Bionetworks/infra-utils/v1.0.9/aws/set_env_vars_file.ps1" - mode: "0664" + mode: "000664" commands: 01_set_env_vars: command: !Join @@ -145,10 +145,10 @@ Resources: files: 'c:\scripts\install-ms-vc.ps1': source: "https://raw.githubusercontent.com/Sage-Bionetworks/infra-utils/v1.0.8/aws/install-ms-vc.ps1" - mode: "0664" + mode: "000664" 'c:\\scripts\\install-jc-agent.ps1': source: "https://raw.githubusercontent.com/TheJumpCloud/support/master/scripts/windows/InstallWindowsAgent.ps1" - mode: "0664" + mode: "000664" commands: 01_install_ms_vc: command: !Join @@ -220,8 +220,6 @@ Resources: 'Fn::FindInMap': [AccountToImportParams, !Ref "AWS::AccountId", VPCId] SecurityGroupIngress: - CidrIp: "10.50.0.0/16" - FromPort: -1 - ToPort: -1 IpProtocol: "-1" Outputs: WindowsInstancePrivateIpAddress: diff --git a/templates/s3/sc-s3-synapse-ra.yaml b/templates/s3/sc-s3-synapse-ra.yaml index 6c7cf61..50a67e3 100644 --- a/templates/s3/sc-s3-synapse-ra.yaml +++ b/templates/s3/sc-s3-synapse-ra.yaml @@ -84,6 +84,7 @@ Resources: config: ignore_checks: - E3001 + - E3006 Properties: Target: Bucket: !Ref S3Bucket