Python Build #2385
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # The following workflow provides an opinionated template you can customize for your own needs. | |
| # | |
| # If you are not an Octopus user, the "Push to Octopus", "Generate Octopus Deploy build information", | |
| # and "Create Octopus Release" steps can be safely deleted. | |
| # | |
| # To configure Octopus, set the OCTOPUS_API_TOKEN secret to the Octopus API key, and | |
| # set the OCTOPUS_SERVER_URL secret to the Octopus URL. | |
| # | |
| # Double check the "project" and "deploy_to" properties in the "Create Octopus Release" step | |
| # match your Octopus projects and environments. | |
| # | |
| # Get a trial Octopus instance from https://octopus.com/start | |
| name: Python Build | |
| 'on': | |
| workflow_dispatch: { } | |
| push: | |
| paths-ignore: | |
| - '.octopus/**' | |
| schedule: | |
| - cron: '0 0 * * *' | |
| jobs: | |
| validate: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Set up Python | |
| uses: actions/setup-python@v5 | |
| with: | |
| # Need 3.11 because of https://stackoverflow.com/questions/77364550/attributeerror-module-pkgutil-has-no-attribute-impimporter-did-you-mean | |
| python-version: 3.11 | |
| - uses: actions/checkout@v4 | |
| - name: Install tools | |
| run: | | |
| pip install pytest coverage setuptools radon xenon | |
| sudo apt-get install -y pycodestyle mypy | |
| - name: Linting | |
| shell: bash | |
| run: find . -path ./domain/sanitizers/stringlifier -prune -o -path ./venv -prune -o -name "*.py" -print0 | xargs -0 -n1 pycodestyle --ignore=E501,W503,E126 | |
| - name: Complexity Enforcement | |
| shell: bash | |
| run: find . -path ./domain/sanitizers/stringlifier -prune -o -path ./tests -prune -o -path ./venv -prune -o -path ./.venv -prune -o -name "*.py" -print0 | xargs -0 -n1 xenon --max-absolute C | |
| test: | |
| name: Test | |
| runs-on: ubuntu-latest | |
| needs: validate | |
| strategy: | |
| matrix: | |
| group: [ 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15 ] | |
| steps: | |
| - name: Install terraform | |
| uses: hashicorp/setup-terraform@v3 | |
| - name: Set up Python | |
| uses: actions/setup-python@v5 | |
| with: | |
| # Need 3.11 because of https://stackoverflow.com/questions/77364550/attributeerror-module-pkgutil-has-no-attribute-impimporter-did-you-mean | |
| python-version: 3.11 | |
| - uses: actions/checkout@v4 | |
| - name: Start Azurite | |
| run: docker run -d -p 10000:10000 -p 10001:10001 -p 10002:10002 mcr.microsoft.com/azure-storage/azurite | |
| shell: bash | |
| - uses: robinraju/[email protected] | |
| with: | |
| repository: "OctopusSolutionsEngineering/OctopusTerraformExport" | |
| latest: true | |
| fileName: "octoterra_linux_amd64_azure.zip" | |
| - uses: robinraju/[email protected] | |
| with: | |
| repository: "OctopusSolutionsEngineering/OctopusRecommendationEngine" | |
| latest: true | |
| fileName: "octolint_linux_amd64_azure.zip" | |
| - name: Start Octoterra | |
| run: | | |
| # Start the octoterra microservice | |
| mkdir octoterra | |
| mv octoterra_linux_amd64_azure.zip octoterra | |
| cd octoterra | |
| unzip octoterra_linux_amd64_azure.zip | |
| chmod +x octoterra_linux_amd64_azure | |
| ./octoterra_linux_amd64_azure & | |
| env: | |
| # The port used when running the octoterra microservice | |
| FUNCTIONS_CUSTOMHANDLER_PORT: 8081 | |
| shell: bash | |
| - name: Start Octolint | |
| run: | | |
| # Start the octolint microservice | |
| mkdir octolint | |
| mv octolint_linux_amd64_azure.zip octolint | |
| cd octolint | |
| unzip octolint_linux_amd64_azure.zip | |
| chmod +x octolint_linux_amd64_azure | |
| ./octolint_linux_amd64_azure & | |
| env: | |
| # The port used when running the octolint microservice | |
| FUNCTIONS_CUSTOMHANDLER_PORT: 8082 | |
| shell: bash | |
| - name: Install Dependencies | |
| run: pip install -r requirements.txt | |
| shell: bash | |
| # To run tests locally, use these instructions: | |
| # 1. source <venv>/bin/activate | |
| # 2. pip install coverage pytest | |
| # 3. pip install -r requirements.txt | |
| # 4. coverage run --omit="./domain/sanitizers/stringlifier/*" -m pytest --junitxml=results.xml | |
| - name: Test | |
| run: | | |
| pip install pytest pytest-split coverage setuptools | |
| PYTHONPATH=$(pwd) coverage run --omit="./tests/*" --omit="./domain/sanitizers/stringlifier/*" -m pytest --junitxml=results.xml --splits 15 --group ${{ matrix.group }} | |
| # See https://github.com/pytest-dev/pytest/issues/2393 | |
| ret=$? | |
| if [ "$ret" = 5 ]; then | |
| echo "No tests collected. Exiting with 0 (instead of 5)." | |
| exit 0 | |
| fi | |
| ls -la | |
| ls -la tests | |
| exit "$ret" | |
| env: | |
| # These are the details of Slack | |
| SLACK_CLIENT_ID: ${{ secrets.SLACK_CLIENT_ID }} | |
| SLACK_CLIENT_REDIRECT: ${{ secrets.SLACK_CLIENT_REDIRECT }} | |
| # These are the details for zendesk | |
| ZENDESK_EMAIL: ${{ secrets.ZENDESK_EMAIL }} | |
| ZENDESK_TOKEN: ${{ secrets.ZENDESK_TOKEN }} | |
| # These are the connection details to Azure OpenAI | |
| OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} | |
| OPENAI_ENDPOINT: ${{ secrets.OPENAI_ENDPOINT }} | |
| # This is the deployment used for function calling | |
| OPENAI_API_DEPLOYMENT_FUNCTIONS: gpt-4o-mini | |
| # This is the deployment used for answering prompts | |
| OPENAI_API_DEPLOYMENT_QUERY: gpt4o | |
| # This is a base 64 encoded version of the Octopus license | |
| LICENSE: ${{ secrets.OCTOPUS_LICENSE }} | |
| # This is the slack webhook used to send messages | |
| SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} | |
| # This is the connection string to Azurite, the test Aure Storage emulator | |
| AzureWebJobsStorage: DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;TableEndpoint=http://127.0.0.1:10002/devstoreaccount1; | |
| # This is the list of admin users | |
| APPLICATION_USERS_ADMIN: "[160104]" | |
| # This is the test user that is used to simulate a login | |
| TEST_GH_USER: "160104" | |
| # This is the GitHub token passed to the chat tests | |
| GH_TEST_TOKEN: ${{ secrets.GH_TEST_TOKEN }} | |
| # This is the encryption salt used when saving encrypted values to Azure Storage | |
| ENCRYPTION_PASSWORD: password | |
| # This is the encryption salt used when saving encrypted values to Azure Storage | |
| ENCRYPTION_SALT: salt | |
| # The location of the octoterra microservice | |
| APPLICATION_OCTOTERRA_URL: http://localhost:8081 | |
| # The location of the octolint microservice | |
| APPLICATION_OCTOLINT_URL: http://localhost:8082 | |
| GITHUB_CLIENT_REDIRECT: https://aiagent.octopus.com/api/oauth_callback | |
| GITHUB_CLIENT_ID: 4ae1ef506301902a41a5 | |
| shell: /usr/bin/bash --noprofile --norc {0} | |
| - name: Upload coverage | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: coverage${{ matrix.group }} | |
| include-hidden-files: true | |
| path: .coverage | |
| - name: Upload test artifacts | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: junit-test-summary-${{ matrix.group }} | |
| path: results.xml | |
| retention-days: 1 | |
| test-results: | |
| needs: test | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Set up Python | |
| uses: actions/setup-python@v5 | |
| with: | |
| # Need 3.11 because of https://stackoverflow.com/questions/77364550/attributeerror-module-pkgutil-has-no-attribute-impimporter-did-you-mean | |
| python-version: 3.11 | |
| - uses: actions/checkout@v4 | |
| - name: Download artifacts | |
| uses: actions/download-artifact@v4 | |
| - name: Run coverage | |
| run: | | |
| pip install pytest coverage setuptools | |
| coverage combine coverage*/.coverage* | |
| coverage xml --omit="./domain/sanitizers/stringlifier/*" | |
| coverage html --omit="./domain/sanitizers/stringlifier/*" --fail-under=80 | |
| - name: Upload combined coverage | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: coverage | |
| include-hidden-files: true | |
| path: .coverage | |
| - name: Upload coverage xml | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: coverage.xml | |
| include-hidden-files: true | |
| path: coverage.xml | |
| - name: Coverage Badge | |
| uses: tj-actions/coverage-badge-py@v2 | |
| - name: Verify Changed files | |
| uses: tj-actions/verify-changed-files@v17 | |
| id: verify-changed-files | |
| with: | |
| files: coverage.svg | |
| - name: Commit files | |
| if: steps.verify-changed-files.outputs.files_changed == 'true' | |
| run: | | |
| git config --local user.email "github-actions[bot]@users.noreply.github.com" | |
| git config --local user.name "github-actions[bot]" | |
| git add coverage.svg | |
| git commit -m "Updated coverage.svg" | |
| - name: Push changes | |
| if: steps.verify-changed-files.outputs.files_changed == 'true' | |
| uses: ad-m/github-push-action@master | |
| continue-on-error: true | |
| with: | |
| github_token: ${{ secrets.github_token }} | |
| branch: ${{ github.ref }} | |
| - uses: actions/setup-node@v3 | |
| with: | |
| node-version: 16 | |
| - name: Install junit-report-merger | |
| run: npm install -g junit-report-merger | |
| - name: Merge reports | |
| run: > | |
| jrm ./junit-test-summary.xml | |
| "junit-test-summary-1/*.xml" | |
| "junit-test-summary-2/*.xml" | |
| "junit-test-summary-3/*.xml" | |
| "junit-test-summary-4/*.xml" | |
| "junit-test-summary-5/*.xml" | |
| "junit-test-summary-6/*.xml" | |
| "junit-test-summary-7/*.xml" | |
| "junit-test-summary-8/*.xml" | |
| "junit-test-summary-9/*.xml" | |
| "junit-test-summary-10/*.xml" | |
| "junit-test-summary-11/*.xml" | |
| "junit-test-summary-12/*.xml" | |
| "junit-test-summary-13/*.xml" | |
| "junit-test-summary-14/*.xml" | |
| "junit-test-summary-15/*.xml" | |
| - if: always() | |
| name: Report | |
| uses: dorny/test-reporter@v1 | |
| with: | |
| name: Python Tests | |
| path: junit-test-summary.xml | |
| reporter: java-junit | |
| fail-on-error: 'false' | |
| build: | |
| name: Build | |
| needs: test-results | |
| runs-on: ubuntu-latest | |
| # group: ubuntu-large-runner | |
| # labels: 4core-ubuntu-runner | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: '0' | |
| - name: Set up Python | |
| uses: actions/setup-python@v5 | |
| with: | |
| # Need 3.11 because of https://stackoverflow.com/questions/77364550/attributeerror-module-pkgutil-has-no-attribute-impimporter-did-you-mean | |
| python-version: 3.11 | |
| - name: Install GitVersion | |
| uses: gittools/actions/gitversion/[email protected] | |
| with: | |
| versionSpec: 5.x | |
| - id: determine_version | |
| name: Determine Version | |
| uses: gittools/actions/gitversion/[email protected] | |
| with: | |
| overrideConfig: mode=Mainline | |
| - name: List Dependencies | |
| run: pip install pipdeptree; pipdeptree > dependencies.txt | |
| shell: bash | |
| - name: Collect Dependencies | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: Dependencies | |
| path: dependencies.txt | |
| - name: List Dependency Updates | |
| run: python3 -m pip list --outdated --format=json | jq -r '.[] | .name+"="+.latest_version' > dependencyUpdates.txt || true | |
| shell: bash | |
| - name: Collect Dependency Updates | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: Dependencies Updates | |
| path: dependencyUpdates.txt | |
| - name: Generate SBOM | |
| if: ${{ github.event_name != 'schedule' }} | |
| run: | | |
| python -m pip install cyclonedx-bom | |
| cyclonedx-py environment > bom.json | |
| shell: bash | |
| - name: Package | |
| if: ${{ github.event_name != 'schedule' }} | |
| run: |- | |
| zip -R OctopusCopilot.${{ steps.determine_version.outputs.semVer }}.zip \ | |
| '*.py' \ | |
| '*.pyc' \ | |
| '*.html' \ | |
| '*.htm' \ | |
| '*.css' \ | |
| '*.js' \ | |
| '*.min' \ | |
| '*.map' \ | |
| '*.sql' \ | |
| '*.png' \ | |
| '*.jpg' \ | |
| '*.jpeg' \ | |
| '*.gif' \ | |
| '*.json' \ | |
| '*.env' \ | |
| '*.txt' \ | |
| '*.Procfile' \ | |
| '*.bestType' \ | |
| '*.conf' \ | |
| '*.encodings' \ | |
| '*.graphql' \ | |
| -x "octoterra/*" \ | |
| -x "**/__pycache__/*" \ | |
| -x ".vscode/*" \ | |
| -x "tests/*" \ | |
| shell: bash | |
| - name: Tag Release | |
| if: ${{ github.ref == 'refs/heads/main' && github.event_name != 'schedule' }} | |
| uses: mathieudutour/[email protected] | |
| with: | |
| custom_tag: ${{ steps.determine_version.outputs.semVer }} | |
| github_token: ${{ secrets.GITHUB_TOKEN }} | |
| - id: create_release | |
| name: Create Release | |
| if: ${{ github.ref == 'refs/heads/main' && github.event_name != 'schedule' }} | |
| uses: softprops/action-gh-release@v1 | |
| with: | |
| tag_name: ${{ steps.determine_version.outputs.semVer }}+run${{ github.run_number }}-attempt${{ github.run_attempt }} | |
| release_name: Release ${{ steps.determine_version.outputs.semVer }} Run ${{ github.run_number }} Attempt ${{ github.run_attempt }} | |
| draft: ${{ github.ref == 'refs/heads/main' && 'false' || 'true' }} | |
| name: Release ${{ steps.determine_version.outputs.semVer }} Run ${{ github.run_number }} Attempt ${{ github.run_attempt }} | |
| - name: Upload Release Asset | |
| if: ${{ github.ref == 'refs/heads/main' && github.event_name != 'schedule' }} | |
| uses: softprops/action-gh-release@v1 | |
| with: | |
| tag_name: ${{ steps.determine_version.outputs.semVer }}+run${{ github.run_number }}-attempt${{ github.run_attempt }} | |
| files: OctopusCopilot.${{ steps.determine_version.outputs.semVer }}.zip | |
| - name: Push packages to Octopus Deploy | |
| if: ${{ github.event_name != 'schedule' }} | |
| uses: OctopusDeploy/push-package-action@v3 | |
| env: | |
| OCTOPUS_API_KEY: ${{ secrets.COPILOT_OCTOPUS_API }} | |
| OCTOPUS_URL: ${{ secrets.COPILOT_OCTOPUS_URL }} | |
| OCTOPUS_SPACE: ${{ secrets.COPILOT_OCTOPUS_SPACE }} | |
| with: | |
| packages: OctopusCopilot.${{ steps.determine_version.outputs.semVer }}.zip | |
| overwrite_mode: OverwriteExisting | |
| - name: Push build information to Octopus Deploy | |
| if: ${{ github.event_name != 'schedule' }} | |
| uses: OctopusDeploy/push-build-information-action@v3 | |
| env: | |
| OCTOPUS_API_KEY: ${{ secrets.COPILOT_OCTOPUS_API }} | |
| OCTOPUS_URL: ${{ secrets.COPILOT_OCTOPUS_URL }} | |
| OCTOPUS_SPACE: ${{ secrets.COPILOT_OCTOPUS_SPACE }} | |
| with: | |
| packages: OctopusCopilot | |
| version: ${{ steps.determine_version.outputs.semVer }} | |
| overwrite_mode: OverwriteExisting | |
| - name: Create Octopus Release | |
| if: ${{ github.ref == 'refs/heads/main' && github.event_name != 'schedule' }} | |
| uses: OctopusDeploy/create-release-action@v3 | |
| env: | |
| OCTOPUS_API_KEY: ${{ secrets.COPILOT_OCTOPUS_API }} | |
| OCTOPUS_URL: ${{ secrets.COPILOT_OCTOPUS_URL }} | |
| OCTOPUS_SPACE: ${{ secrets.COPILOT_OCTOPUS_SPACE }} | |
| with: | |
| project: Octopus Copilot Function | |
| packages: OctopusCopilot:${{ steps.determine_version.outputs.semVer }} | |
| release_number: ${{ steps.determine_version.outputs.semVer }}+${{ steps.determine_version.outputs.ShortSha }}.${{ github.run_number }}.${{ github.run_attempt }} | |
| git_ref: main | |
| release_notes: | | |
| * GitHub Owner: OctopusSolutionsEngineering | |
| * GitHub Repo: OctopusCopilot | |
| * GitHub Workflow: build.yaml | |
| * GitHub Sha: ${{ github.sha }} | |
| * GitHub Run: ${{ github.run_number }} | |
| * GitHub Attempt: ${{ github.run_attempt }} | |
| * GitHub Run ID: ${{ github.run_id }} | |
| Here are the notes for the packages | |
| #{each package in Octopus.Release.Package} | |
| - #{package.PackageId} #{package.Version} | |
| #{each commit in package.Commits} | |
| - [#{commit.CommitId}](#{commit.LinkUrl}) - #{commit.Comment} | |
| #{/each} | |
| #{each workItem in package.WorkItems} | |
| - [#{workItem.Id}](#{workItem.LinkUrl}) - #{workItem.Description} | |
| #{/each} | |
| #{/each} | |
| permissions: | |
| id-token: write | |
| checks: write | |
| contents: write |