Update CI

Author: mdellweg

.bumpversion.cfg

@@ -1,14 +1,5 @@
 #!/bin/sh
 
-# This file is shared between some projects please keep all copies in sync
-# Known places:
-#   - https://github.com/pulp/pulp-cli/blob/main/.ci/run_container.sh
-#   - https://github.com/pulp/pulp-cli-deb/blob/main/.ci/run_container.sh
-#   - https://github.com/pulp/pulp-cli-gem/blob/main/.ci/run_container.sh
-#   - https://github.com/pulp/pulp-cli-maven/blob/main/.ci/run_container.sh
-#   - https://github.com/pulp/pulp-cli-ostree/blob/main/.ci/run_container.sh
-#   - https://github.com/pulp/squeezer/blob/develop/tests/run_container.sh
-
 set -eu
 
 BASEPATH="$(dirname "$(readlink -f "$0")")"

.ci/run_container.sh

@@ -1,13 +1,16 @@
+#!/bin/env python3
+
 import itertools
 import os
 import re
+import tomllib
 
-import toml
 from git import GitCommandError, Repo
 from packaging.version import parse as parse_version
 
 # Read Towncrier settings
-tc_settings = toml.load("pyproject.toml")["tool"]["towncrier"]
+with open("pyproject.toml", "rb") as fp:
+    tc_settings = tomllib.load(fp)["tool"]["towncrier"]
 
 CHANGELOG_FILE = tc_settings.get("filename", "NEWS.rst")
 START_STRING = tc_settings.get(
@@ -21,23 +24,32 @@
 TITLE_FORMAT = tc_settings.get("title_format", "{name} {version} ({project_date})")
 
 
+# Build a regex to find the header of a changelog section.
+# It must have a single capture group to single out the version.
+# see help(re.split) for more info.
 NAME_REGEX = r".*"
-VERSION_REGEX = r"([0-9]+\.[0-9]+\.[0-9][0-9ab]*)"
+VERSION_REGEX = r"[0-9]+\.[0-9]+\.[0-9][0-9ab]*"
+VERSION_CAPTURE_REGEX = rf"({VERSION_REGEX})"
 DATE_REGEX = r"[0-9]{4}-[0-9]{2}-[0-9]{2}"
 TITLE_REGEX = (
     "("
     + re.escape(
         TITLE_FORMAT.format(name="NAME_REGEX", version="VERSION_REGEX", project_date="DATE_REGEX")
     )
     .replace("NAME_REGEX", NAME_REGEX)
+    .replace("VERSION_REGEX", VERSION_CAPTURE_REGEX, 1)
     .replace("VERSION_REGEX", VERSION_REGEX)
     .replace("DATE_REGEX", DATE_REGEX)
     + ")"
 )
 
 
 def get_changelog(repo, branch):
-    return repo.git.show(f"{branch}:{CHANGELOG_FILE}") + "\n"
+    branch_tc_settings = tomllib.loads(repo.git.show(f"{branch}:pyproject.toml"))["tool"][
+        "towncrier"
+    ]
+    branch_changelog_file = branch_tc_settings.get("filename", "NEWS.rst")
+    return repo.git.show(f"{branch}:{branch_changelog_file}") + "\n"
 
 
 def _tokenize_changes(splits):
@@ -89,7 +101,7 @@ def main():
             for change in main_changes:
                 fp.write(change[1])
 
-        repo.git.commit("-m", "Update Changelog", "-m" "[noissue]", CHANGELOG_FILE)
+        repo.git.commit("-m", "Update Changelog", CHANGELOG_FILE)
 
 
 if __name__ == "__main__":

.ci/scripts/collect_changes.py

@@ -10,7 +10,7 @@ then
   exit 1
 fi
 
-NEW_BRANCH="$(bump2version --dry-run --list release | sed -Ene 's/^new_version=([[:digit:]]+\.[[:digit:]]+)\..*$/\1/p')"
+NEW_BRANCH="$(bump-my-version show new_version --increment release | sed -Ene 's/^([[:digit:]]+\.[[:digit:]]+)\.[[:digit:]]+$/\1/p')"
 
 if [[ -z "${NEW_BRANCH}" ]]
 then
@@ -23,6 +23,6 @@ git branch "${NEW_BRANCH}"
 # Clean changelog snippets.
 find CHANGES/ \( -name "*.feature" -o -name "*.bugfix" -o -name "*.removal" -o -name "*.doc" -o -name "*.translation" -o -name "*.devel" -o -name "*.misc" \) -exec git rm -f \{\} +
 
-bump2version minor --commit --message $'Bump version to {new_version}\n\n[noissue]' --allow-dirty
+bump-my-version bump minor --commit --message $'Bump version to {new_version}' --allow-dirty
 
 git push origin "${NEW_BRANCH}"

.ci/scripts/create_release_branch.sh

@@ -0,0 +1,57 @@
+#!/bin/env python3
+
+# This script is running with elevated privileges from the main branch against pull requests.
+
+import re
+import sys
+import tomllib
+from pathlib import Path
+
+from git import Repo
+
+
+def main():
+    assert len(sys.argv) == 3
+
+    with open("pyproject.toml", "rb") as fp:
+        PYPROJECT_TOML = tomllib.load(fp)
+    BLOCKING_REGEX = re.compile(r"DRAFT|WIP|NO\s*MERGE|DO\s*NOT\s*MERGE|EXPERIMENT")
+    ISSUE_REGEX = re.compile(r"(?:fixes|closes)[\s:]+#(\d+)")
+    CHERRY_PICK_REGEX = re.compile(r"^\s*\(cherry picked from commit [0-9a-f]*\)\s*$")
+    CHANGELOG_EXTS = [
+        f".{item['directory']}" for item in PYPROJECT_TOML["tool"]["towncrier"]["type"]
+    ]
+
+    repo = Repo(".")
+
+    base_commit = repo.commit(sys.argv[1])
+    head_commit = repo.commit(sys.argv[2])
+
+    pr_commits = list(repo.iter_commits(f"{base_commit}..{head_commit}"))
+
+    labels = {
+        "multi-commit": len(pr_commits) > 1,
+        "cherry-pick": False,
+        "no-issue": False,
+        "no-changelog": False,
+        "wip": False,
+    }
+    for commit in pr_commits:
+        labels["wip"] |= BLOCKING_REGEX.search(commit.summary) is not None
+        no_issue = ISSUE_REGEX.search(commit.message, re.IGNORECASE) is None
+        labels["no-issue"] |= no_issue
+        cherry_pick = CHERRY_PICK_REGEX.search(commit.message) is not None
+        labels["cherry-pick"] |= cherry_pick
+        changelog_snippets = [
+            k
+            for k in commit.stats.files
+            if k.startswith("CHANGES/") and Path(k).suffix in CHANGELOG_EXTS
+        ]
+        labels["no-changelog"] |= not changelog_snippets
+
+    print("ADD_LABELS=" + ",".join((k for k, v in labels.items() if v)))
+    print("REMOVE_LABELS=" + ",".join((k for k, v in labels.items() if not v)))
+
+
+if __name__ == "__main__":
+    main()

.ci/scripts/pr_labels.py

@@ -10,7 +10,7 @@ then
   exit 1
 fi
 
-NEW_VERSION="$(bump2version --dry-run --list release | sed -ne 's/^new_version=//p')"
+NEW_VERSION="$(bump-my-version show new_version --increment release)"
 echo "Release ${NEW_VERSION}"
 
 if ! [[ "${NEW_VERSION}" == "${BRANCH}"* ]]
@@ -20,7 +20,7 @@ then
 fi
 
 towncrier build --yes --version "${NEW_VERSION}"
-bump2version release --commit --message "Release {new_version}" --tag --tag-name "{new_version}" --tag-message "Release {new_version}" --allow-dirty
-bump2version patch --commit
+bump-my-version bump release --commit --message "Release {new_version}" --tag --tag-name "{new_version}" --tag-message "Release {new_version}" --allow-dirty
+bump-my-version bump patch --commit
 
 git push origin "${BRANCH}" "${NEW_VERSION}"

.ci/scripts/release.sh

@@ -2,11 +2,13 @@
 import re
 import subprocess
 import sys
+import tomllib
 from pathlib import Path
 
-import toml
 from github import Github
 
+with open("pyproject.toml", "rb") as fp:
+    PYPROJECT_TOML = tomllib.load(fp)
 KEYWORDS = ["fixes", "closes"]
 BLOCKING_REGEX = [
     "DRAFT",
@@ -15,14 +17,15 @@
     r"DO\s*NOT\s*MERGE",
     "EXPERIMENT",
 ]
-NO_ISSUE = "[noissue]"
-CHANGELOG_EXTS = [
-    f".{item['directory']}" for item in toml.load("pyproject.toml")["tool"]["towncrier"]["type"]
-]
+CHANGELOG_EXTS = [f".{item['directory']}" for item in PYPROJECT_TOML["tool"]["towncrier"]["type"]]
+NOISSUE_MARKER = "[noissue]"
 
 sha = sys.argv[1]
 message = subprocess.check_output(["git", "log", "--format=%B", "-n 1", sha]).decode("utf-8")
 
+if NOISSUE_MARKER in message:
+    sys.exit("Do not add '[noissue]' in the commit message.")
+
 if any((re.match(pattern, message) for pattern in BLOCKING_REGEX)):
     sys.exit("This PR is not ready for consumption.")
 
@@ -61,13 +64,5 @@ def check_changelog(issue):
         if not cherry_pick:
             check_status(issue)
             check_changelog(issue)
-else:
-    if NO_ISSUE in message:
-        print("Commit {sha} has no issues but is tagged {tag}.".format(sha=sha[0:7], tag=NO_ISSUE))
-    else:
-        sys.exit(
-            "Error: no attached issues found for {sha}. If this was intentional, add "
-            " '{tag}' to the commit message.".format(sha=sha[0:7], tag=NO_ISSUE)
-        )
 
 print("Commit message for {sha} passed.".format(sha=sha[0:7]))

.ci/scripts/validate_commit_message.py

@@ -4,13 +4,13 @@ updates:
   directory: "/"
   schedule:
     interval: daily
-  open-pull-requests-limit: 10
   commit-message:
-    prefix: "[noissue]"
+    prefix: "[PIP] "
+  open-pull-requests-limit: 10
 - package-ecosystem: github-actions
   directory: "/"
   schedule:
-    interval: daily
-  open-pull-requests-limit: 10
+    interval: weekly
   commit-message:
-    prefix: "[noissue]"
+    prefix: "[GHA] "
+  open-pull-requests-limit: 10

.flake8

@@ -18,26 +18,10 @@ jobs:
     steps:
       - name: "Checkout repository"
         uses: "actions/checkout@v4"
-      - uses: "actions/cache@v4"
-        with:
-          path: "~/.cache/pip"
-          key: "${{ runner.os }}-pip-${{ hashFiles('**/*requirements.txt', '**/*constraints.lock', '**/setup.py', '**/pyproject.toml') }}"
-          restore-keys: |
-            ${{ runner.os }}-pip-
-
-      - name: "Set up Python"
-        uses: "actions/setup-python@v5"
-        with:
-          python-version: "3.11"
-      - name: "Manually install from sources"
-        run: |
-          python -m pip install -e . -e ./pulp-glue-maven
-          echo "CODEQL_PYTHON=$(which python)" >> "$GITHUB_ENV"
       - name: "Initialize CodeQL"
         uses: "github/codeql-action/init@v3"
         with:
           languages: "python"
-          setup-python-dependencies: false
 
       - name: "Perform CodeQL Analysis"
         uses: "github/codeql-action/analyze@v3"

.github/dependabot.yml

@@ -20,7 +20,7 @@ jobs:
           git config user.email [email protected]
       - name: "Collect changes"
         run: |
-          pip install GitPython packaging toml
+          pip install GitPython packaging
           python3 .ci/scripts/collect_changes.py
       - name: "Create Pull Request"
         uses: "peter-evans/create-pull-request@v6"

.github/workflows/codeql.yml

@@ -15,41 +15,45 @@ concurrency:
 jobs:
   single_commit:
     runs-on: "ubuntu-latest"
-    name: "Label multiple commit PR"
+    name: "Label PR"
     permissions:
       pull-requests: "write"
     steps:
       - uses: "actions/checkout@v4"
         with:
           fetch-depth: 0
-      - name: "Commit Count Check"
+      - uses: "actions/setup-python@v5"
+        with:
+          python-version: "3.12"
+      - name: "Determine PR labels"
         run: |
+          pip install GitPython==3.1.42
           git fetch origin ${{ github.event.pull_request.head.sha }}
-          echo "COMMIT_COUNT=$(git log  --oneline --no-merges origin/${{ github.base_ref }}..${{ github.event.pull_request.head.sha }} | wc -l)" >> "$GITHUB_ENV"
+          python .ci/scripts/pr_labels.py "origin/${{ github.base_ref }}" "${{ github.event.pull_request.head.sha }}" >> "$GITHUB_ENV"
       - uses: "actions/github-script@v7"
+        name: "Apply PR Labels"
         with:
           script: |
-            const labelName = "multi-commit";
-            const { COMMIT_COUNT } = process.env;
+            const { ADD_LABELS, REMOVE_LABELS } = process.env;
 
-            if (COMMIT_COUNT == 1)
-            {
-              try {
-                await github.rest.issues.removeLabel({
-                  issue_number: context.issue.number,
-                  owner: context.repo.owner,
-                  repo: context.repo.repo,
-                  name: labelName,
-                });
-              } catch(err) {
+            if (REMOVE_LABELS.length) {
+              for await (const labelName of REMOVE_LABELS.split(",")) {
+                try {
+                  await github.rest.issues.removeLabel({
+                    issue_number: context.issue.number,
+                    owner: context.repo.owner,
+                    repo: context.repo.repo,
+                    name: labelName,
+                  });
+                } catch(err) {
+                }
               }
             }
-            else
-            {
+            if (ADD_LABELS.length) {
               await github.rest.issues.addLabels({
                 issue_number: context.issue.number,
                 owner: context.repo.owner,
                 repo: context.repo.repo,
-                labels: [labelName],
+                labels: ADD_LABELS.split(","),
               });
             }

.github/workflows/collect_changes.yml

@@ -18,7 +18,7 @@ jobs:
       - name: "Install dependencies"
         run: |
           python -m pip install --upgrade pip
-          pip install bump2version towncrier~=23.11.0
+          pip install bump-my-version~=0.20.0 towncrier~=23.11.0
       - name: "Setup git"
         run: |
           git config user.name pulpbot

.github/workflows/pr_checks.yml

@@ -18,7 +18,7 @@ jobs:
           git config user.email [email protected]
       - name: "Install python dependencies"
         run: |
-          pip install bump2version
+          pip install bump-my-version~=0.20.0
       - name: "Create Release Branch"
         run: |
           .ci/scripts/create_release_branch.sh