ci.yml

name: CI

on:
  push:
    branches-ignore:
      - 'main'
      - 'docs'

env:
  BUILDER_VERSION: v0.9.84
  BUILDER_SOURCE: releases
  BUILDER_HOST: https://d19elf31gohf1l.cloudfront.net
  PACKAGE_NAME: aws-iot-device-sdk-python-v2
  LINUX_BASE_IMAGE: ubuntu-16-x64
  RUN: ${{ github.run_id }}-${{ github.run_number }}
  AWS_DEFAULT_REGION: us-east-1
  DA_TOPIC: test/da
  DA_SHADOW_PROPERTY: datest
  DA_SHADOW_VALUE_SET: ON
  DA_SHADOW_VALUE_DEFAULT: OFF
  CI_UTILS_FOLDER: "./aws-iot-device-sdk-python-v2/utils"
  CI_SAMPLES_CFG_FOLDER: "./aws-iot-device-sdk-python-v2/.github/workflows"
  CI_FLEET_PROVISIONING_ROLE: arn:aws:iam::180635532705:role/service-role/CI_FleetProvisioning_Role
  CI_GREENGRASS_INSTALLER_ROLE: arn:aws:iam::180635532705:role/CI_GreengrassInstaller_Role
  CI_MQTT5_ROLE: arn:aws:iam::180635532705:role/CI_MQTT5_Role
  CI_BUILD_AND_TEST_ROLE: arn:aws:iam::180635532705:role/V2_SDK_Unit_Testing
  CI_JOBS_SERVICE_CLIENT_ROLE: arn:aws:iam::180635532705:role/CI_JobsServiceClient_Role
  CI_SHADOW_SERVICE_CLIENT_ROLE: arn:aws:iam::180635532705:role/CI_ShadowServiceClient_Role

jobs:

  windows:
    runs-on: windows-2022
    permissions:
      id-token: write # This is required for requesting the JWT
    steps:
    - name: Install boto3
      run: |
        python -m pip install boto3
    - name: configure AWS credentials (containers)
      uses: aws-actions/configure-aws-credentials@v4
      with:
        role-to-assume: ${{ env.CI_BUILD_AND_TEST_ROLE }}
        aws-region: ${{ env.AWS_DEFAULT_REGION }}
    - name: Build ${{ env.PACKAGE_NAME }}
      run: |
        python -c "from urllib.request import urlretrieve; urlretrieve('${{ env.BUILDER_HOST }}/${{ env.BUILDER_SOURCE }}/${{ env.BUILDER_VERSION }}/builder.pyz?run=${{ env.RUN }}', 'builder.pyz')"
        python builder.pyz build -p ${{ env.PACKAGE_NAME }}
    - name: configure AWS credentials (MQTT5)
      uses: aws-actions/configure-aws-credentials@v4
      with:
        role-to-assume: ${{ env.CI_MQTT5_ROLE }}
        aws-region: ${{ env.AWS_DEFAULT_REGION }}
    - name: Service tests
      shell: bash
      run: |
        cd aws-iot-device-sdk-python-v2
        python3 -m pip install .
        source utils/test_setup.sh s3://iot-sdk-ci-bucket-us-east1/IotUsProdMqtt5EnvironmentVariables.txt us-east-1
        python3 -m unittest test.test_shadow
        python3 -m unittest test.test_jobs
        python3 -m unittest test.test_identity
        source utils/test_cleanup.sh
        cd ..

  osx:
    runs-on: ${{ matrix.runner }}
    strategy:
      fail-fast: false
      matrix:
        runner:
          - macos-latest
    permissions:
      id-token: write # This is required for requesting the JWT
    steps:
    - name: Install boto3
      # Starting from macos14, boto3 is no longer available in macos homebrew python https://github.com/Homebrew/homebrew-core/issues/157500
      # use virtual env to work around.
      run: |
        python3 -m venv .venv
        source .venv/bin/activate
        python3 -m pip install boto3
    - name: configure AWS credentials (containers)
      uses: aws-actions/configure-aws-credentials@v4
      with:
        role-to-assume: ${{ env.CI_BUILD_AND_TEST_ROLE }}
        aws-region: ${{ env.AWS_DEFAULT_REGION }}
    - name: Build ${{ env.PACKAGE_NAME }}
      run: |
        source .venv/bin/activate
        python3 -c "from urllib.request import urlretrieve; urlretrieve('${{ env.BUILDER_HOST }}/${{ env.BUILDER_SOURCE }}/${{ env.BUILDER_VERSION }}/builder.pyz?run=${{ env.RUN }}', 'builder')"
        chmod a+x builder
        ./builder build -p ${{ env.PACKAGE_NAME }}
    - name: configure AWS credentials (MQTT5)
      uses: aws-actions/configure-aws-credentials@v4
      with:
        role-to-assume: ${{ env.CI_MQTT5_ROLE }}
        aws-region: ${{ env.AWS_DEFAULT_REGION }}
    - name: Service tests
      shell: bash
      run: |
        source .venv/bin/activate
        cd aws-iot-device-sdk-python-v2
        python3 -m pip install .
        source utils/test_setup.sh s3://iot-sdk-ci-bucket-us-east1/IotUsProdMqtt5EnvironmentVariables.txt us-east-1
        python3 -m unittest test.test_shadow
        python3 -m unittest test.test_jobs
        python3 -m unittest test.test_identity
        source utils/test_cleanup.sh
        cd ..

  linux:
    runs-on: ubuntu-22.04 # latest
    permissions:
      id-token: write # This is required for requesting the JWT
    steps:
    - name: Install AWS SDK for Python
      run: |
        python3 -m pip install boto3
    - name: configure AWS credentials (containers)
      uses: aws-actions/configure-aws-credentials@v4
      with:
        role-to-assume: ${{ env.CI_BUILD_AND_TEST_ROLE }}
        aws-region: ${{ env.AWS_DEFAULT_REGION }}
    - name: Build ${{ env.PACKAGE_NAME }}
      run: |
        python3 -c "from urllib.request import urlretrieve; urlretrieve('${{ env.BUILDER_HOST }}/${{ env.BUILDER_SOURCE }}/${{ env.BUILDER_VERSION }}/builder.pyz?run=${{ env.RUN }}', 'builder')"
        chmod a+x builder
        ./builder build -p ${{ env.PACKAGE_NAME }}
    - name: configure AWS credentials (MQTT5)
      uses: aws-actions/configure-aws-credentials@v4
      with:
        role-to-assume: ${{ env.CI_MQTT5_ROLE }}
        aws-region: ${{ env.AWS_DEFAULT_REGION }}
    - name: Service tests
      shell: bash
      run: |
        cd aws-iot-device-sdk-python-v2
        python3 -m pip install .
        source utils/test_setup.sh s3://iot-sdk-ci-bucket-us-east1/IotUsProdMqtt5EnvironmentVariables.txt us-east-1
        python3 -m unittest test.test_shadow
        python3 -m unittest test.test_jobs
        python3 -m unittest test.test_identity
        source utils/test_cleanup.sh
        cd ..

  # Runs the service client tests
  linux-service-client-tests:
    runs-on: ubuntu-22.04
    permissions:
      id-token: write # This is required for requesting the JWT
    steps:
      - name: CI setup
        run: |
          python3 -m pip install boto3
          sudo apt-get update -y
          sudo apt-get install softhsm -y
          softhsm2-util --version
      - name: configure AWS credentials (containers)
        uses: aws-actions/configure-aws-credentials@v4
        with:
          role-to-assume: ${{ env.CI_BUILD_AND_TEST_ROLE }}
          aws-region: ${{ env.AWS_DEFAULT_REGION }}
      - name: Build ${{ env.PACKAGE_NAME }}
        run: |
          python3 -c "from urllib.request import urlretrieve; urlretrieve('${{ env.BUILDER_HOST }}/${{ env.BUILDER_SOURCE }}/${{ env.BUILDER_VERSION }}/builder.pyz?run=${{ env.RUN }}', 'builder')"
          chmod a+x builder
          ./builder build -p ${{ env.PACKAGE_NAME }}
      - name: configure AWS credentials (service tests Fleet Provisioning)
        uses: aws-actions/configure-aws-credentials@v4
        with:
          role-to-assume: ${{ env.CI_FLEET_PROVISIONING_ROLE}}
          aws-region: ${{ env.AWS_DEFAULT_REGION }}
      - name: run MQTT3 Fleet Provisioning
        working-directory: ./aws-iot-device-sdk-python-v2/servicetests
        run: |
          export PYTHONPATH=${{ github.workspace }}/aws-iot-device-sdk-python-v2/utils:${{ github.workspace }}/aws-iot-device-sdk-python-v2/samples
          python3 ./test_cases/test_fleet_provisioning.py --config-file test_cases/mqtt3_fleet_provisioning_cfg.json --thing-name-prefix Fleet_Thing_
      - name: run MQTT5 Fleet Provisioning
        working-directory: ./aws-iot-device-sdk-python-v2/servicetests
        run: |
          export PYTHONPATH=${{ github.workspace }}/aws-iot-device-sdk-python-v2/utils:${{ github.workspace }}/aws-iot-device-sdk-python-v2/samples
          python3 ./test_cases/test_fleet_provisioning.py --config-file test_cases/mqtt5_fleet_provisioning_cfg.json --thing-name-prefix Fleet_Thing_
      - name: run MQTT3 Fleet Provisioning with CSR
        working-directory: ./aws-iot-device-sdk-python-v2/servicetests
        run: |
          export PYTHONPATH=${{ github.workspace }}/aws-iot-device-sdk-python-v2/utils:${{ github.workspace }}/aws-iot-device-sdk-python-v2/samples
          python3 ./test_cases/test_fleet_provisioning.py --config-file test_cases/mqtt3_fleet_provisioning_with_csr_cfg.json --thing-name-prefix Fleet_Thing_
      - name: run MQTT5 Fleet Provisioning with CSR
        working-directory: ./aws-iot-device-sdk-python-v2/servicetests
        run: |
          export PYTHONPATH=${{ github.workspace }}/aws-iot-device-sdk-python-v2/utils:${{ github.workspace }}/aws-iot-device-sdk-python-v2/samples
          python3 ./test_cases/test_fleet_provisioning.py --config-file test_cases/mqtt5_fleet_provisioning_with_csr_cfg.json --thing-name-prefix Fleet_Thing_
      - name: configure AWS credentials (service tests Shadow)
        uses: aws-actions/configure-aws-credentials@v4
        with:
          role-to-assume: ${{ env.CI_SHADOW_SERVICE_CLIENT_ROLE }}
          aws-region: ${{ env.AWS_DEFAULT_REGION }}
      - name: run MQTT5 Shadow Update
        working-directory: ./aws-iot-device-sdk-python-v2/servicetests
        run: |
          export PYTHONPATH=${{ github.workspace }}/aws-iot-device-sdk-python-v2/utils:${{ github.workspace }}/aws-iot-device-sdk-python-v2/samples
          python3 ./test_cases/test_shadow_update.py --config-file test_cases/mqtt5_shadow_cfg.json
      - name: run MQTT3 Shadow Update
        working-directory: ./aws-iot-device-sdk-python-v2/servicetests
        run: |
          export PYTHONPATH=${{ github.workspace }}/aws-iot-device-sdk-python-v2/utils:${{ github.workspace }}/aws-iot-device-sdk-python-v2/samples
          python3 ./test_cases/test_shadow_update.py --config-file test_cases/mqtt3_shadow_cfg.json
      - name: run MQTT5 Named Shadow Update
        working-directory: ./aws-iot-device-sdk-python-v2/servicetests
        run: |
          export PYTHONPATH=${{ github.workspace }}/aws-iot-device-sdk-python-v2/utils:${{ github.workspace }}/aws-iot-device-sdk-python-v2/samples
          python3 ./test_cases/test_shadow_update.py --config-file test_cases/mqtt5_named_shadow_cfg.json
      - name: run MQTT3 Named Shadow Update
        working-directory: ./aws-iot-device-sdk-python-v2/servicetests
        run: |
          export PYTHONPATH=${{ github.workspace }}/aws-iot-device-sdk-python-v2/utils:${{ github.workspace }}/aws-iot-device-sdk-python-v2/samples
          python3 ./test_cases/test_shadow_update.py --config-file test_cases/mqtt3_named_shadow_cfg.json
      - name: configure AWS credentials (service tests Jobs)
        uses: aws-actions/configure-aws-credentials@v4
        with:
          role-to-assume: ${{ env.CI_JOBS_SERVICE_CLIENT_ROLE}}
          aws-region: ${{ env.AWS_DEFAULT_REGION }}
      - name: run MQTT3 Jobs servicetests
        working-directory: ./aws-iot-device-sdk-python-v2/servicetests
        run: |
          export PYTHONPATH=${{ github.workspace }}/aws-iot-device-sdk-python-v2/utils:${{ github.workspace }}/aws-iot-device-sdk-python-v2/samples
          python3 ./test_cases/test_jobs_execution.py --config-file test_cases/mqtt3_jobs_cfg.json
      - name: run MQTT5 Jobs servicetests
        working-directory: ./aws-iot-device-sdk-python-v2/servicetests
        run: |
          export PYTHONPATH=${{ github.workspace }}/aws-iot-device-sdk-python-v2/utils:${{ github.workspace }}/aws-iot-device-sdk-python-v2/samples
          python3 ./test_cases/test_jobs_execution.py --config-file test_cases/mqtt5_jobs_cfg.json

  # check that docs can still build
  check-docs:
    runs-on: ubuntu-22.04 # latest
    steps:
      - uses: actions/checkout@v2
        with:
          submodules: true
      - name: Check docs
        run: |
          python3 -m pip install sphinx
          python3 -m pip install --verbose .
          ./make-docs.py

  check-codegen-edits:
    runs-on: ubuntu-22.04 # latest
    steps:
      - uses: actions/checkout@v2
        with:
          fetch-depth: 0
      - name: Check for edits to code-generated files
        run: |
          ./utils/check_codegen_edits.py