Migrate Data #148
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Migrate Data | |
| on: | |
| workflow_dispatch: | |
| inputs: | |
| applyTo: | |
| required: true | |
| default: Development | |
| type: choice | |
| description: Where do you want to migrate data? | |
| options: | |
| - Development | |
| - Staging/UAT | |
| - Production | |
| jobs: | |
| build: | |
| runs-on: ubuntu-22.04 | |
| environment: ${{ inputs.applyTo == 'Production' && 'Production' || 'Development'}} | |
| env: | |
| DATABASE_PASSWORD: ${{ secrets.DATABASE_PASSWORD }} | |
| GOOGLE_API_KEY: ${{ secrets.GOOGLE_API_KEY }} | |
| AWS_VPC_ID: ${{ secrets.AWS_VPC_ID }} | |
| AWS_SUBNET_PRIMARY: ${{ secrets.AWS_SUBNET_PRIMARY }} | |
| AWS_ZONE_PRIMARY: ${{ secrets.AWS_ZONE_PRIMARY }} | |
| AWS_SUBNET_SECONDARY0: ${{ secrets.AWS_SUBNET_SECONDARY0 }} | |
| AWS_ZONE_SECONDARY0: ${{ secrets.AWS_ZONE_SECONDARY0 }} | |
| AWS_SUBNET_SECONDARY1: ${{ secrets.AWS_SUBNET_SECONDARY1 }} | |
| AWS_ZONE_SECONDARY1: ${{ secrets.AWS_ZONE_SECONDARY1 }} | |
| AWS_SSH_KEY: ${{ secrets.AWS_SSH_KEY }} | |
| GIT_REPOSITORY_URL: https://github.com/neu-dsg/dailp-encoding | |
| OAUTH_TOKEN: ${{ secrets.OAUTH_TOKEN }} | |
| RUST_LOG: info | |
| BASTION_IP: ${{ secrets.TEST_BASTION_IP }} | |
| TF_STAGE: ${{ inputs.applyTo == 'Development' && 'dev' || ( inputs.applyTo == 'Staging/UAT' && 'uat' || 'prod' )}} | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v3 | |
| - name: Install Nix | |
| uses: cachix/install-nix-action@v20 | |
| with: | |
| nix_path: nixpkgs=channel:nixos-21.11 | |
| - name: Use binary cache for nix store | |
| uses: cachix/cachix-action@v12 | |
| with: | |
| name: dailp | |
| # If you chose API tokens for write access OR if you have a private cache | |
| authToken: "${{ secrets.CACHIX_AUTH_TOKEN }}" | |
| pushFilter: "(-dailp$|-dailp-|-terraform-config$|-source$|\\.tar\\.gz$|-output$|-plan$|-apply-now$|-apply$)" | |
| - name: Configure AWS credentials | |
| uses: aws-actions/configure-aws-credentials@v2 | |
| with: | |
| aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} | |
| aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} | |
| aws-region: us-east-1 | |
| - name: Build project | |
| run: | | |
| nix build --impure -L | |
| nix run --impure -L .#tf-init | |
| SECURITY_GROUP_ID=$(nix run --impure .#tf-output access_security_group_id) | |
| echo "ACCESS_SECURITY_GROUP=$SECURITY_GROUP_ID" >> $GITHUB_ENV | |
| # - name: Validate spreadsheets | |
| # run: nix run --impure -L .#validate-data | |
| - name: Add public IP to AWS security group | |
| uses: sohelamin/aws-security-group-add-ip-action@master | |
| with: | |
| aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} | |
| aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} | |
| aws-region: us-east-1 | |
| aws-security-group-id: ${{ env.ACCESS_SECURITY_GROUP }} | |
| port: 22 | |
| to-port: 22 | |
| protocol: tcp | |
| - name: Migrate from spreadsheets to database | |
| # Port forward 5432 to remote database | |
| run: | | |
| echo "Creating SSH key file..." | |
| echo "${{secrets.AWS_BASTION_SSH_KEY}}" > dailp-deployment-key.pem | |
| chmod 400 dailp-deployment-key.pem | |
| echo "Retrieving terraform outputs..." | |
| DATABASE_ENDPOINT=$(nix run --impure .#tf-output database_endpoint) | |
| echo "Configuring SSH client..." | |
| mkdir -p ~/.ssh | |
| ssh-keyscan -H $BASTION_IP >> ~/.ssh/known_hosts | |
| echo "Forwarding port 5432 to remote database" | |
| ssh -i dailp-deployment-key.pem -f -N -L 5432:$DATABASE_ENDPOINT ec2-user@$BASTION_IP | |
| export DAILP_API_URL=$(nix run --impure .#tf-output functions_url) | |
| export DATABASE_URL=postgres://dailp:$DATABASE_PASSWORD@localhost:5432/dailp | |
| export CF_URL=$(nix run --impure .#tf-output cloudfront_distro_url) | |
| nix run --impure -L .#migrate-data | |
| - name: Publish website | |
| run: | | |
| curl -X POST -d {} "$(nix run --impure .#tf-output amplify_webhook)" -H "Content-Type:application/json" |