Merge pull request #821 from nautobot/release-3.8.0

Author: jdrew82

.cookiecutter.json

@@ -21,7 +21,7 @@
         "_drift_manager": {
             "template": "https://github.com/nautobot/cookiecutter-nautobot-app.git",
             "template_dir": "nautobot-app",
-            "template_ref": "refs/tags/nautobot-app-v2.4.2",
+            "template_ref": "refs/tags/nautobot-app-v2.5.0",
             "cookie_dir": "",
             "branch_prefix": "drift-manager",
             "pull_request_strategy": "create",
@@ -30,7 +30,7 @@
                 "poetry"
             ],
             "draft": false,
-            "baked_commit_ref": "a9ba090fb8fb08d173c3c720a6dbe92a59f4c6f8"
+            "baked_commit_ref": "a93c33a2e5583173d90b69dbc7268ca90083a248"
         }
     }
 }

.github/pull_request_template.md

@@ -27,7 +27,7 @@
     Please feel free to update todos to keep track of your own notes for WIP PRs.
 -->
 - [ ] Explanation of Change(s)
-- [ ] Added change log fragment(s) (for more information see [the documentation](https://docs.nautobot.com/projects/core/en/stable/development/#creating-changelog-fragments))
+- [ ] Added change log fragment(s) (for more information see [the documentation](https://docs.nautobot.com/projects/core/en/stable/development/core/#creating-changelog-fragments))
 - [ ] Attached Screenshots, Payload Example
 - [ ] Unit, Integration Tests
 - [ ] Documentation Updates (when adding/changing features)

.github/workflows/ci.yml

@@ -81,12 +81,26 @@ jobs:
           poetry-version: "1.8.5"
       - name: "Linting: yamllint"
         run: "poetry run invoke yamllint"
+  markdownlint:
+    runs-on: "ubuntu-22.04"
+    env:
+      INVOKE_NAUTOBOT_SSOT_LOCAL: "True"
+    steps:
+      - name: "Check out repository code"
+        uses: "actions/checkout@v4"
+      - name: "Setup environment"
+        uses: "networktocode/gh-action-setup-poetry-environment@v6"
+        with:
+          poetry-version: "1.8.5"
+      - name: "Linting: markdownlint"
+        run: "poetry run invoke markdownlint"
   check-in-docker:
     needs:
       - "ruff-format"
       - "ruff-lint"
       - "poetry"
       - "yamllint"
+      - "markdownlint"
     runs-on: "ubuntu-22.04"
     strategy:
       fail-fast: true
@@ -111,9 +125,9 @@ jobs:
         run: "poetry run invoke lock --constrain-nautobot-ver --constrain-python-ver"
       - name: "Set up Docker Buildx"
         id: "buildx"
-        uses: "docker/setup-buildx-action@v3"
+        uses: "docker/setup-buildx-action@b5ca514318bd6ebac0fb2aedd5d36ec1b5c232a2"  # v3.10.0
       - name: "Build"
-        uses: "docker/build-push-action@v5"
+        uses: "docker/build-push-action@ca052bb54ab0790a636c9b5f226502c73d547a25"  # v5.4.0
         with:
           builder: "${{ steps.buildx.outputs.name }}"
           context: "./"
@@ -127,8 +141,6 @@ jobs:
             NAUTOBOT_VER=${{ matrix.nautobot-version }}
             PYTHON_VER=${{ matrix.python-version }}
             CI=true
-      - name: "Copy credentials"
-        run: "cp development/creds.example.env development/creds.env"
       - name: "Linting: pylint"
         run: "poetry run invoke pylint"
       - name: "Checking: App Config"
@@ -141,7 +153,7 @@ jobs:
     strategy:
       fail-fast: true
       matrix:
-        python-version: ["3.9", "3.12"]
+        python-version: ["3.9"]  # 3.12 stable is tested in unittest_report stage.
         db-backend: ["postgresql"]
         nautobot-version: ["stable"]
         include:
@@ -170,9 +182,60 @@ jobs:
         run: "poetry run invoke lock --constrain-nautobot-ver --constrain-python-ver"
       - name: "Set up Docker Buildx"
         id: "buildx"
-        uses: "docker/setup-buildx-action@v3"
+        uses: "docker/setup-buildx-action@b5ca514318bd6ebac0fb2aedd5d36ec1b5c232a2"  # v3.10.0
+      - name: "Build"
+        uses: "docker/build-push-action@ca052bb54ab0790a636c9b5f226502c73d547a25"  # v5.4.0
+        with:
+          builder: "${{ steps.buildx.outputs.name }}"
+          context: "./"
+          push: false
+          load: true
+          tags: "${{ env.APP_NAME }}/nautobot:${{ matrix.nautobot-version }}-py${{ matrix.python-version }}"
+          file: "./development/Dockerfile"
+          cache-from: "type=gha,scope=${{ matrix.nautobot-version }}-py${{ matrix.python-version }}"
+          cache-to: "type=gha,scope=${{ matrix.nautobot-version }}-py${{ matrix.python-version }}"
+          build-args: |
+            NAUTOBOT_VER=${{ matrix.nautobot-version }}
+            PYTHON_VER=${{ matrix.python-version }}
+            CI=true
+      - name: "Use Mysql invoke settings when needed"
+        run: "cp invoke.mysql.yml invoke.yml"
+        if: "matrix.db-backend == 'mysql'"
+      - name: "Run Tests"
+        run: "poetry run invoke unittest"
+
+  unittest_report:
+    needs:
+      - "check-in-docker"
+    strategy:
+      fail-fast: true
+      matrix:
+        python-version: ["3.12"]
+        db-backend: ["postgresql"]
+        nautobot-version: ["stable"]
+    runs-on: "ubuntu-22.04"
+    permissions:
+      pull-requests: "write"
+      contents: "write"
+    env:
+      INVOKE_NAUTOBOT_SSOT_PYTHON_VER: "${{ matrix.python-version }}"
+      INVOKE_NAUTOBOT_SSOT_NAUTOBOT_VER: "${{ matrix.nautobot-version }}"
+    steps:
+      - name: "Check out repository code"
+        uses: "actions/checkout@v4"
+      - name: "Setup environment"
+        uses: "networktocode/gh-action-setup-poetry-environment@v6"
+        with:
+          poetry-version: "1.8.5"
+      - name: "Constrain Nautobot version and regenerate lock file"
+        env:
+          INVOKE_NAUTOBOT_SSOT_LOCAL: "true"
+        run: "poetry run invoke lock --constrain-nautobot-ver --constrain-python-ver"
+      - name: "Set up Docker Buildx"
+        id: "buildx"
+        uses: "docker/setup-buildx-action@b5ca514318bd6ebac0fb2aedd5d36ec1b5c232a2"  # v3.10.0
       - name: "Build"
-        uses: "docker/build-push-action@v5"
+        uses: "docker/build-push-action@ca052bb54ab0790a636c9b5f226502c73d547a25"  # v5.4.0
         with:
           builder: "${{ steps.buildx.outputs.name }}"
           context: "./"
@@ -192,7 +255,28 @@ jobs:
         run: "cp invoke.mysql.yml invoke.yml"
         if: "matrix.db-backend == 'mysql'"
       - name: "Run Tests"
-        run: "poetry run invoke unittest"
+        run: "poetry run invoke unittest --coverage"
+      - name: "Generate Coverage Comment"
+        if: >
+          contains(fromJson('["develop","ltm-1.6"]'), github.base_ref) &&
+          (github.head_ref != 'main') && (!startsWith(github.head_ref, 'release'))
+        id: "coverage_comment"
+        uses: "py-cov-action/python-coverage-comment-action@d1ff8fbb5ff80feedb3faa0f6d7b424f417ad0e1"  # v3.30
+        with:
+          GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}"
+          MINIMUM_GREEN: 90
+          MINIMUM_ORANGE: 80
+          ANNOTATE_MISSING_LINES: true
+          ANNOTATION_TYPE: "warning"
+      - name: "Store Pull Request comment to be posted"
+        if: >
+          contains(fromJson('["develop","ltm-1.6"]'), github.base_ref) &&
+          (github.head_ref != 'main') && (!startsWith(github.head_ref, 'release'))
+        uses: "actions/upload-artifact@v4"
+        with:
+          name: "python-coverage-comment-action"
+          path: "python-coverage-comment-action.txt"
+
   changelog:
     if: >
       contains(fromJson('["develop","ltm-1.6"]'), github.base_ref) &&
@@ -214,41 +298,39 @@ jobs:
   publish_gh:
     needs:
       - "unittest"
+      - "unittest_report"
     name: "Publish to GitHub"
     runs-on: "ubuntu-22.04"
     if: "startsWith(github.ref, 'refs/tags/v')"
     env:
       INVOKE_NAUTOBOT_SSOT_LOCAL: "True"
+    permissions:
+      contents: "write"
     steps:
       - name: "Check out repository code"
         uses: "actions/checkout@v4"
-      - name: "Set up Python"
-        uses: "actions/setup-python@v5"
+      - name: "Setup environment"
+        uses: "networktocode/gh-action-setup-poetry-environment@v6"
         with:
+          poetry-version: "1.8.5"
           python-version: "3.12"
-      - name: "Install Python Packages"
-        run: "pip install poetry"
+          poetry-install-options: "--no-root"
       - name: "Set env"
         run: "echo RELEASE_VERSION=${GITHUB_REF:10} >> $GITHUB_ENV"
       - name: "Run Poetry Version"
         run: "poetry version $RELEASE_VERSION"
-      - name: "Install Dependencies (needed for mkdocs)"
-        run: "poetry install --no-root"
       - name: "Build Documentation"
         run: "poetry run invoke build-and-check-docs"
       - name: "Run Poetry Build"
         run: "poetry build"
       - name: "Upload binaries to release"
-        uses: "svenstaro/upload-release-action@v2"
-        with:
-          repo_token: "${{ secrets.GH_NAUTOBOT_BOT_TOKEN }}"
-          file: "dist/*"
-          tag: "${{ github.ref }}"
-          overwrite: true
-          file_glob: true
+        run: "gh release upload ${{ github.ref_name }} dist/*.{tar.gz,whl}"
+        env:
+          GH_TOKEN: "${{ secrets.GITHUB_TOKEN }}"
   publish_pypi:
     needs:
       - "unittest"
+      - "unittest_report"
     name: "Push Package to PyPI"
     runs-on: "ubuntu-22.04"
     if: "startsWith(github.ref, 'refs/tags/v')"
@@ -257,24 +339,22 @@ jobs:
     steps:
       - name: "Check out repository code"
         uses: "actions/checkout@v4"
-      - name: "Set up Python"
-        uses: "actions/setup-python@v5"
+      - name: "Setup environment"
+        uses: "networktocode/gh-action-setup-poetry-environment@v6"
         with:
+          poetry-version: "1.8.5"
           python-version: "3.12"
-      - name: "Install Python Packages"
-        run: "pip install poetry"
+          poetry-install-options: "--no-root"
       - name: "Set env"
         run: "echo RELEASE_VERSION=${GITHUB_REF:10} >> $GITHUB_ENV"
       - name: "Run Poetry Version"
         run: "poetry version $RELEASE_VERSION"
-      - name: "Install Dependencies (needed for mkdocs)"
-        run: "poetry install --no-root"
       - name: "Build Documentation"
         run: "poetry run invoke build-and-check-docs"
       - name: "Run Poetry Build"
         run: "poetry build"
       - name: "Push to PyPI"
-        uses: "pypa/gh-action-pypi-publish@release/v1"
+        uses: "pypa/gh-action-pypi-publish@76f52bc884231f62b9a034ebfe128415bbaabdfc"  # v1.12.4
         with:
           user: "__token__"
           password: "${{ secrets.PYPI_API_TOKEN }}"
@@ -295,7 +375,7 @@ jobs:
         # ENVs cannot be used directly in job.if. This is a workaround to check
         # if SLACK_WEBHOOK_URL is present.
         if: "env.SLACK_WEBHOOK_URL != ''"
-        uses: "slackapi/slack-github-action@v1"
+        uses: "slackapi/slack-github-action@fcfb566f8b0aab22203f066d80ca1d7e4b5d05b3"  # v1.27.1
         with:
           payload: |
             {

.github/workflows/coverage.yml

@@ -0,0 +1,36 @@
+---
+name: "Post coverage comment"
+
+on: # yamllint disable-line rule:truthy rule:comments
+  workflow_run:
+    workflows: ["CI"]
+    types:
+      - "completed"
+
+jobs:
+  test:
+    name: "Post coverage comment to PR"
+    runs-on: "ubuntu-latest"
+    if: github.event.workflow_run.event == 'pull_request' && github.event.workflow_run.conclusion == 'success' # yamllint disable-line rule:quoted-strings rule:comments
+    permissions:
+      # Gives the action the necessary permissions for publishing new
+      # comments in pull requests.
+      pull-requests: "write"
+      # Gives the action the necessary permissions for editing existing
+      # comments (to avoid publishing multiple comments in the same PR)
+      contents: "write" # yamllint disable-line rule:indentation rule:comments
+      # Gives the action the necessary permissions for looking up the
+      # workflow that launched this workflow, and download the related
+      # artifact that contains the comment to be published
+      actions: "read"
+    steps:
+      # DO NOT run actions/checkout here, for security reasons
+      # For details, refer to https://securitylab.github.com/research/github-actions-preventing-pwn-requests/
+      - name: "Post comment"
+        uses: "py-cov-action/python-coverage-comment-action@d1ff8fbb5ff80feedb3faa0f6d7b424f417ad0e13"  # v3.30
+        with:
+          GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}"
+          GITHUB_PR_RUN_ID: "${{ github.event.workflow_run.id }}"
+          # Update those if you changed the default values:
+          # COMMENT_ARTIFACT_NAME: python-coverage-comment-action
+          # COMMENT_FILENAME: python-coverage-comment-action.txt

.gitignore

@@ -56,6 +56,7 @@ coverage.xml
 *.py,cover
 .hypothesis/
 .pytest_cache/
+lcov.info
 
 # Translations
 *.mo

development/creds.example.env

@@ -53,7 +53,8 @@ NAUTOBOT_APIC_VERIFY_DEVNET=False
 
 SERVICENOW_PASSWORD="changeme"
 
-IPFABRIC_API_TOKEN=secrettoken
+NAUTOBOT_SSOT_IPFABRIC_HOST="https://demo.ipfabric.io"
+NAUTOBOT_SSOT_IPFABRIC_API_TOKEN=secrettoken
 
 MERAKI_ORG_ID='123456'
 MERAKI_TOKEN='vtx01710aa0fn452740055y1hs60ns8c107ho168'

development/development.env

@@ -38,6 +38,9 @@ MYSQL_USER=${NAUTOBOT_DB_USER}
 MYSQL_DATABASE=${NAUTOBOT_DB_NAME}
 MYSQL_ROOT_HOST=%
 
+# Use a less verbose log level for Celery Beat
+NAUTOBOT_BEAT_LOG_LEVEL=INFO
+
 NAUTOBOT_HOST="http://nautobot:8080"
 
 NAUTOBOT_CELERY_TASK_SOFT_TIME_LIMIT=7200
@@ -60,10 +63,10 @@ NAUTOBOT_SSOT_ACI_COMMENTS="Created by ACI SSoT Integration"
 NAUTOBOT_SSOT_ENABLE_ARISTACV="False"
 NAUTOBOT_ARISTACV_CONTROLLER_SITE=""
 NAUTOBOT_ARISTACV_CREATE_CONTROLLER="True"
-NAUTOBOT_ARISTACV_CVAAS_URL="www.arista.io:443"
+NAUTOBOT_ARISTACV_CVAAS_URL="www.cv-prod-euwest-2.arista.io"
 NAUTOBOT_ARISTACV_CVP_HOST=""
 NAUTOBOT_ARISTACV_CVP_PORT="443"
-NAUTOBOT_ARISTACV_CVP_USERNAME="changeme"
+NAUTOBOT_ARISTACV_CVP_USERNAME="networktocode"
 NAUTOBOT_ARISTACV_DELETE_ON_SYNC="False"
 NAUTOBOT_ARISTACV_IMPORT_ACTIVE="False"
 NAUTOBOT_ARISTACV_IMPORT_TAG="False"
@@ -108,9 +111,9 @@ SERVICENOW_INSTANCE=""
 SERVICENOW_USERNAME=""
 
 NAUTOBOT_SSOT_ENABLE_IPFABRIC="False"
-IPFABRIC_HOST="https://ipfabric.example.com"
-IPFABRIC_SSL_VERIFY="True"
-IPFABRIC_TIMEOUT=15
+NAUTOBOT_SSOT_IPFABRIC_HOST="https://ipfabric.example.com"
+NAUTOBOT_SSOT_IPFABRIC_SSL_VERIFY="True"
+NAUTOBOT_SSOT_IPFABRIC_TIMEOUT=15
 
 NAUTOBOT_SSOT_ENABLE_ITENTIAL="True"
 
@@ -123,4 +126,3 @@ NAUTOBOT_SSOT_LIBRENMS_SYSTEM_OF_RECORD="LibreNMS"
 NAUTOBOT_SSOT_LIBRENMS_HOSTNAME_FIELD="sysName" # hostname or sysName
 
 NAUTOBOT_SSOT_ENABLE_SOLARWINDS="False"
-

development/docker-compose.base.yml

@@ -41,8 +41,8 @@ services:
   beat:
     entrypoint:
       - "sh"
-      - "-c"  # this is to evaluate the $NAUTOBOT_LOG_LEVEL from the env
-      - "nautobot-server celery beat -l $$NAUTOBOT_LOG_LEVEL"  ## $$ because of docker-compose
+      - "-c"  # this is to evaluate the $NAUTOBOT_BEAT_LOG_LEVEL from the env
+      - "nautobot-server celery beat -l $$NAUTOBOT_BEAT_LOG_LEVEL"  ## $$ because of docker-compose
     depends_on:
       nautobot:
         condition: "service_healthy"

development/docker-compose.dev.yml

@@ -40,8 +40,8 @@ services:
   beat:
     entrypoint:
       - "sh"
-      - "-c"  # this is to evaluate the $NAUTOBOT_LOG_LEVEL from the env
-      - "watchmedo auto-restart --directory './' --pattern '*.py' --recursive -- nautobot-server celery beat -l $$NAUTOBOT_LOG_LEVEL"  ## $$ because of docker-compose
+      - "-c"  # this is to evaluate the $NAUTOBOT_BEAT_LOG_LEVEL from the env
+      - "watchmedo auto-restart --directory './' --pattern '*.py' --recursive -- nautobot-server celery beat -l $$NAUTOBOT_BEAT_LOG_LEVEL"  ## $$ because of docker-compose
     volumes:
       - "./nautobot_config.py:/opt/nautobot/nautobot_config.py"
       - "../:/source"

development/docker-compose.mysql.yml

@@ -22,7 +22,7 @@ services:
       - "creds.env"
       - "development_mysql.env"
   db:
-    image: "mysql:8"
+    image: "mysql:lts"
     command:
       - "--max_connections=1000"
     env_file: