circle.yml

version: 2.1

orbs:
  aws-s3: circleci/aws-s3@1.0.11

jobs:
  frontend-test:
    docker:
      - image: circleci/node:12
    working_directory: ~/project/frontend
    steps:
      - checkout:
          path: ~/project
      - restore_cache:
          keys:
            - frontend-deps-{{ .Branch }}-{{ checksum "package-lock.json" }}
            - frontend-deps-{{ .Branch }}-
            - frontend-deps-
      - run:
          name: Build and install Ocim API client
          command: ./scripts/build-api-client.sh
      - run:
          name: Installing frontend dependencies
          command: npm install
      - save_cache:
          paths:
            - node_modules
          key: frontend-deps-{{ .Branch }}-{{ checksum "package-lock.json" }}
      - run:
          name: Run frontend tests
          command: npm test -- --reporters=default --reporters=jest-junit --ci --runInBand
      - store_test_results:
          path: .

  frontend-quality:
    docker:
      - image: circleci/node:12
    working_directory: ~/project/frontend
    steps:
      - checkout:
          path: ~/project
      - restore_cache:
          keys:
            - frontend-deps-{{ .Branch }}-{{ checksum "package-lock.json" }}
            - frontend-deps-{{ .Branch }}-
            - frontend-deps-
      - run:
          name: Build and install Ocim API client
          command: ./scripts/build-api-client.sh
      - run:
          name: Installing frontend dependencies
          command: npm install
      - save_cache:
          paths:
            - node_modules
          key: frontend-deps-{{ .Branch }}-{{ checksum "package-lock.json" }}
      - run:
          name: Run frontend quality tests
          command: npm run lint
      - store_test_results:
          path: .

  frontend-coverage:
      docker:
        - image: circleci/node:12
      working_directory: ~/project/frontend
      steps:
        - checkout:
            path: ~/project
        - restore_cache:
            keys:
              - frontend-deps-{{ .Branch }}-{{ checksum "package-lock.json" }}
              - frontend-deps-{{ .Branch }}-
              - frontend-deps-
        - run:
            name: Build and install Ocim API client
            command: ./scripts/build-api-client.sh
        - run:
            name: Installing frontend dependencies
            command: npm install
        - save_cache:
            paths:
              - node_modules
            key: frontend-deps-{{ .Branch }}-{{ checksum "package-lock.json" }}
        - run:
            name: Generate frontend coverage report
            command: npm run coverage || true
        - store_test_results:
            path: ~/project/frontend/coverage/clover.xml

  frontend-build:
    docker:
      - image: circleci/node:12
    working_directory: ~/project/frontend
    steps:
      - checkout:
          path: ~/project
      - restore_cache:
          keys:
            - frontend-deps-{{ .Branch }}-{{ checksum "package-lock.json" }}
            - frontend-deps-{{ .Branch }}-
            - frontend-deps-
      - run:
          name: Build and install Ocim API client
          command: ./scripts/build-api-client.sh
      - run:
          name: Installing frontend dependencies
          command: npm install
      - save_cache:
          paths:
            - node_modules
          key: frontend-deps-{{ .Branch }}-{{ checksum "package-lock.json" }}
      - run:
          name: Rebuild node-sass to generate bindings
          command: npm rebuild node-sass
      - run:
          name: Render environment variables to .env file
          command: |
            echo $(env | grep REACT_APP) >> .env
      - run:
          name: Build React frontend
          command: npm run build
      - persist_to_workspace:
          root: build
          paths:
            - .

  frontend-deploy:
    docker:
      - image: circleci/python:3.8
    steps:
      - attach_workspace:
          at: frontend-build
      - aws-s3/sync:
          from: frontend-build/
          to: "s3://$AWS_S3_BUCKET/frontend"
          overwrite: true
          # `acl public-read` is needed to make the objects in the bucket publicly accesible.
          arguments: |
            --acl public-read
      - run:
          name: Invalidate CloudFront cache
          command: |
            aws cloudfront create-invalidation --distribution-id $CLOUDFRONT_DISTRIBUTION_ID --paths '/*'

  build:
    docker:
      - image: circleci/python:3.6-stretch-node-browsers
        environment:
          DEBUG: 'true'
          DEFAULT_FORK: 'open-craft/edx-platform'
          LOAD_BALANCER_FRAGMENT_NAME_PREFIX: 'integration-'
          SIMPLE_THEME_SKELETON_THEME_VERSION: 'master'
      - image: redis
      - image: mongo:3.2-jessie
      - image: "circleci/mysql:5"
        environment:
          MYSQL_ALLOW_EMPTY_PASSWORD: True
      - image: circleci/postgres:10.1-alpine
        environment:
          PG_HOST: 127.0.0.1
          POSTGRES_USER: circleci
          POSTGRES_DB: circle_test
    parameters:
        tests_group:
            type: integer
        tests_type:
            type: string
    environment:
        TEST_GROUP: <<parameters.tests_group>>
    steps:
      - checkout
      - restore_cache:
          key: dependencies-{{ checksum "requirements.txt" }}
      - run:
          name: Wait for Redis
          command: dockerize -wait tcp://localhost:6379 -timeout 10s
      - run:
          name: Wait for Postgres
          command: dockerize -wait tcp://localhost:5432 -timeout 10s
      - run:
          name: Wait for MySQL
          command: dockerize -wait tcp://localhost:3306 -timeout 10s
      - run:
          name: Wait for MongoDB
          command: dockerize -wait tcp://localhost:27017 -timeout 10s
      - run:
          name: Install Dependencies
          command: |
            sed -i 's/libmysqlclient/default-libmysqlclient/' debian_packages.lst
            make install_system_dependencies
            sudo ln -s /usr/lib/x86_64-linux-gnu/libmysqlclient.so /usr/lib/x86_64-linux-gnu/libmysqlclient.so.18
            bin/install-supported-firefox
            python3 -m venv venv
            . venv/bin/activate
            pip install --upgrade pip
            pip install --upgrade virtualenv
            pip install -r requirements.txt
            pip install -r cleanup_utils/requirements.txt
            npm install
      - run:
          name: Run Consul
          command: |
            sudo wget -P /tmp https://releases.hashicorp.com/consul/1.2.1/consul_1.2.1_linux_amd64.zip
            sudo unzip /tmp/consul_1.2.1_linux_amd64.zip -d /usr/local/bin
            mkdir -p /tmp/consul-data
            if [[ "<< parameters.tests_type >>" == "test.integration" ]] ; then
              echo Running consul agent to connect to the integration consul server
              consul agent -retry-join $CONSUL_SERVERS -encrypt $CONSUL_ENCRYPT -data-dir /tmp/consul-data
            else
              echo Running a local consul agent
              consul agent -dev
            fi
          background: true
      - save_cache:
          key: dependencies-{{ checksum "requirements.txt" }}
          paths:
            - "venv"
      - run:
          name: Create Database
          command: make create_db
      - run:
          name: Run Tests
          command: |
            . venv/bin/activate
            mkdir -p /tmp/coverage
            make << parameters.tests_type >>
            if [ -e .coverage.* ]; then
                cp -v .coverage.* /tmp/coverage/.
            fi
            if [ -d coverage ]; then
                cp -vR coverage/* /tmp/coverage/.
            fi
          no_output_timeout: 2h
          environment:
            ANSIBLE_HOST_KEY_CHECKING: False
      - persist_to_workspace:
          root: /tmp
          paths:
              - coverage

  e2e-tests:
    docker:
      - image: circleci/python:3.6-stretch-node-browsers
        environment:
          DEBUG: 'true'
          DEFAULT_FORK: 'open-craft/edx-platform'
          LOAD_BALANCER_FRAGMENT_NAME_PREFIX: 'integration-'
      - image: redis
      - image: mongo:3.2-jessie
      - image: "circleci/mysql:5"
        environment:
          MYSQL_ALLOW_EMPTY_PASSWORD: True
      - image: circleci/postgres:10.1-alpine
        environment:
          PG_HOST: 127.0.0.1
          POSTGRES_USER: circleci
          POSTGRES_DB: circle_test
    environment:
        REACT_APP_OCIM_API_BASE: http://localhost:5000
    steps:
      - checkout
      - restore_cache:
          key: dependencies-{{ checksum "requirements.txt" }}
      - run:
          name: Wait for Redis
          command: dockerize -wait tcp://localhost:6379 -timeout 10s
      - run:
          name: Wait for Postgres
          command: dockerize -wait tcp://localhost:5432 -timeout 10s
      - run:
          name: Wait for MySQL
          command: dockerize -wait tcp://localhost:3306 -timeout 10s
      - run:
          name: Wait for MongoDB
          command: dockerize -wait tcp://localhost:27017 -timeout 10s
      - run:
          name: Install Dependencies
          command: |
            sed -i 's/libmysqlclient/default-libmysqlclient/' debian_packages.lst
            make install_system_dependencies
            sudo ln -s /usr/lib/x86_64-linux-gnu/libmysqlclient.so /usr/lib/x86_64-linux-gnu/libmysqlclient.so.18
            bin/install-supported-firefox
            python3 -m venv venv
            . venv/bin/activate
            pip install --upgrade pip
            pip install --upgrade virtualenv
            pip install -r requirements.txt
            pip install -r cleanup_utils/requirements.txt
            npm install
      - run:
          name: Run Consul
          command: |
            sudo wget -P /tmp https://releases.hashicorp.com/consul/1.2.1/consul_1.2.1_linux_amd64.zip
            sudo unzip /tmp/consul_1.2.1_linux_amd64.zip -d /usr/local/bin
            mkdir -p /tmp/consul-data
            consul agent -dev
          background: true
      - save_cache:
          key: dependencies-{{ checksum "requirements.txt" }}
          paths:
            - "venv"
      - run:
          name: Create Database
          command: make create_db
      - run:
          name: Run Tests
          command: |
            . venv/bin/activate
            mkdir -p /tmp/coverage
            make test.e2e
            if [ -e .coverage.* ]; then
                cp -v .coverage.* /tmp/coverage/.
            fi
            if [ -d coverage ]; then
                cp -vR coverage/* /tmp/coverage/.
            fi
          no_output_timeout: 2h
          environment:
            ANSIBLE_HOST_KEY_CHECKING: False
      - persist_to_workspace:
          root: /tmp
          paths:
              - coverage

  coverage:
    docker:
      - image: circleci/python:3.6-stretch-node
    steps:
      - checkout
      - restore_cache:
          key: dependencies-{{ checksum "requirements.txt" }}
      - run:
          name: Install Dependencies
          command: |
            python3 -m venv venv
            . venv/bin/activate
            pip install --upgrade pip
            pip install --upgrade virtualenv
            sudo apt-get update; sudo apt-get install python2.7-dev
            sudo apt-get install mysql-client
            pip install -r requirements.txt
            pip install -r cleanup_utils/requirements.txt
            sudo apt-get install postgresql-client
            sudo apt-get install unzip
            sudo wget -P /tmp https://releases.hashicorp.com/consul/1.2.1/consul_1.2.1_linux_amd64.zip
            sed -i 's/libmysqlclient/default-libmysqlclient/' debian_packages.lst
            make install_system_dependencies
            npm install
      - attach_workspace:
          at: /tmp/workspace
      - run:
          command: |
              . venv/bin/activate
              coverage combine --append /tmp/workspace/coverage/
              make cov.html
              if [ -e /tmp/workspace/coverage/text/coverage.txt ]; then cat /tmp/workspace/coverage/text/coverage.txt; fi

  cleanup:
    docker:
      - image: circleci/python:3.6-stretch
        environment:
          LOAD_BALANCER_FRAGMENT_NAME_PREFIX: 'integration-'
          DJANGO_SETTINGS_MODULE: 'opencraft.settings'
      - image: redis
    steps:
      - checkout
      - restore_cache:
          key: dependencies-{{ checksum "requirements.txt" }}
      - run:
          name: Install Dependencies
          command: |
            python3 -m venv venv
            . venv/bin/activate
            sudo apt-get install libpython2.7-dev
            pip install -r cleanup_utils/requirements.txt
      - run:
          name: Run Consul
          command: |
            echo Running consul agent to connect to the integration consul server
            sudo wget -P /tmp https://releases.hashicorp.com/consul/1.2.1/consul_1.2.1_linux_amd64.zip
            sudo unzip /tmp/consul_1.2.1_linux_amd64.zip -d /usr/local/bin
            mkdir -p /tmp/consul-data
            consul agent -retry-join $CONSUL_SERVERS -encrypt $CONSUL_ENCRYPT -data-dir /tmp/consul-data
          background: true
      - run:
          name: Wait for Redis
          command: dockerize -wait tcp://localhost:6379 -timeout 10s
      - run:
          name: Cleanup
          command: |
            . venv/bin/activate
            make test.integration_cleanup
          no_output_timeout: 20m
          when: always

workflows:
  version: 2
  build-test-and-deploy:
    jobs:
      - build:
          name: quality_migrations_tests
          tests_group: 0
          tests_type: TEST_ENV_FILE=.env.test.ci clean test.quality test.migrations_missing
      - build:
          name: python_unit_tests
          tests_group: 0
          tests_type: TEST_ENV_FILE=.env.test.ci clean test.unit
      - build:
          name: browser_tests
          tests_group: 0
          tests_type: TEST_ENV_FILE=.env.test.ci clean test.browser
      - e2e-tests
      - build:
          name: js_tests
          tests_group: 0
          tests_type: TEST_ENV_FILE=.env.test.ci clean test.js
      - build:
          name: integration_tests_group_1
          tests_group: 1
          tests_type: test.integration
          requires:
            - quality_migrations_tests
            - python_unit_tests
            - browser_tests
            - js_tests
      - build:
          name: integration_tests_group_2
          tests_group: 2
          tests_type: test.integration
          requires:
            - quality_migrations_tests
            - python_unit_tests
            - browser_tests
            - js_tests
      - coverage:
          name: coverage
          requires:
            - integration_tests_group_1
            - integration_tests_group_2

  frontend-build-test-and-deploy:
    jobs:
      - frontend-quality:
          filters:
            tags:
              only: /^release\-\w+\-\w+$/
      - frontend-coverage:
          filters:
            tags:
              only: /^release\-\w+\-\w+$/
      - frontend-test:
          requires:
            - frontend-quality
          filters:
            tags:
              only: /^release\-\w+\-\w+$/
      - frontend-build:
          context: ocim-frontend-stage
          requires:
            - frontend-test
      # Frontend deployment (only on stage branch)
      - frontend-deploy:
          name: frontend-deploy-stage
          context: ocim-frontend-stage
          requires:
            - frontend-build
          filters:
            branches:
              only:
                - stage
      # Deploy production
      - frontend-build:
          name: frontend-build-production
          context: ocim-frontend-production
          requires:
            - frontend-test
          filters:
            branches:
              ignore: /.*/
            tags:
              only: /^release\-\w+\-\w+$/
      - frontend-deploy:
          name: frontend-deploy-production
          context: ocim-frontend-production
          requires:
            - frontend-build-production
          filters:
            branches:
              ignore: /.*/
            tags:
              only: /^release\-\w+\-\w+$/

  # used for periodically cleaning up left-over resources
  scheduled-cleanup:
    triggers:
      - schedule:
          cron: "0 1 * * *"
          filters:
            branches:
              only:
                - master
    jobs:
      - cleanup

  on-demand-cleanup:
    jobs:
      - cleanup:
          filters:
            branches:
              only:
              - ci-cleanup