diff --git a/.github/workflows/update-composer.yml b/.github/workflows/update-composer.yml new file mode 100644 index 00000000..93fac6c7 --- /dev/null +++ b/.github/workflows/update-composer.yml @@ -0,0 +1,51 @@ +# This action updates the osbuild-composer ref in the Schutzfile +--- +name: "Update osbuild-composer commit ID" + +on: # yamllint disable-line rule:truthy + workflow_dispatch: # support manual dispatch + schedule: + # Every Sunday at 12:00 + - cron: "0 12 * * 0" + +jobs: + update-and-push: + runs-on: ubuntu-24.04 + env: + GH_TOKEN: ${{ secrets.SCHUTZBOT_GITHUB_ACCESS_TOKEN }} + steps: + - name: Clone + # This workflow can be started from a branch, but we always want to + # work on main to update the osbuild-composer ref. + # WARNING: This means the update-schutzfile-script below will always be + # run from the main branch. + run: | + git clone --depth=1 --branch main https://github.com/$GITHUB_REPOSITORY ./src + + - name: User config + working-directory: ./src + run: | + git config user.name "schutzbot" + git config user.email "schutzbot@gmail.com" + + - name: Update Schutzfile + working-directory: ./src + # script also creates github_pr_body.txt + run: | + ./schutzbot/update-schutzfile-composer + + - name: Open PR + working-directory: ./src + run: | + if git diff --exit-code; then echo "No changes"; exit 0; fi + branch="schutzfile-composer-$(date -I)" + git checkout -b "${branch}" + git add Schutzfile + git commit -m "schutzfile: Update osbuild-composer dependency commit ID" + remote="https://oauth2:${GH_TOKEN}@github.com/${GITHUB_REPOSITORY}" + git push -f "${remote}" "${branch}" + gh pr create \ + --title "Update osbuild-composer dependency commit ID" \ + --body-file "github_pr_body.txt" \ + --base "main" \ + --head "${branch}" diff --git a/Schutzfile b/Schutzfile index 21250cc0..1acc24aa 100644 --- a/Schutzfile +++ b/Schutzfile @@ -1,6 +1,9 @@ { "fedora-42": { "dependencies": { + "composer": { + "commit": "73e71e29381cd96fe4666091714d0b8c90a62805" + }, "osbuild": { "commit": "8d2a28c5b984667d29aaeca101b5a2b1d72605e6" } @@ -44,6 +47,9 @@ }, "rhel-8.4": { "dependencies": { + "composer": { + "commit": "73e71e29381cd96fe4666091714d0b8c90a62805" + }, "osbuild": { "commit": "8d2a28c5b984667d29aaeca101b5a2b1d72605e6" } @@ -51,6 +57,9 @@ }, "rhel-8.8": { "dependencies": { + "composer": { + "commit": "73e71e29381cd96fe4666091714d0b8c90a62805" + }, "osbuild": { "commit": "8d2a28c5b984667d29aaeca101b5a2b1d72605e6" } @@ -58,6 +67,9 @@ }, "rhel-8.9": { "dependencies": { + "composer": { + "commit": "73e71e29381cd96fe4666091714d0b8c90a62805" + }, "osbuild": { "commit": "8d2a28c5b984667d29aaeca101b5a2b1d72605e6" } @@ -65,6 +77,9 @@ }, "rhel-8.10": { "dependencies": { + "composer": { + "commit": "73e71e29381cd96fe4666091714d0b8c90a62805" + }, "osbuild": { "commit": "8d2a28c5b984667d29aaeca101b5a2b1d72605e6" } @@ -72,6 +87,9 @@ }, "rhel-9.2": { "dependencies": { + "composer": { + "commit": "73e71e29381cd96fe4666091714d0b8c90a62805" + }, "osbuild": { "commit": "8d2a28c5b984667d29aaeca101b5a2b1d72605e6" } @@ -79,6 +97,9 @@ }, "rhel-9.3": { "dependencies": { + "composer": { + "commit": "73e71e29381cd96fe4666091714d0b8c90a62805" + }, "osbuild": { "commit": "8d2a28c5b984667d29aaeca101b5a2b1d72605e6" } @@ -86,6 +107,9 @@ }, "rhel-9.4": { "dependencies": { + "composer": { + "commit": "73e71e29381cd96fe4666091714d0b8c90a62805" + }, "osbuild": { "commit": "8d2a28c5b984667d29aaeca101b5a2b1d72605e6" } @@ -93,6 +117,9 @@ }, "rhel-9.5": { "dependencies": { + "composer": { + "commit": "73e71e29381cd96fe4666091714d0b8c90a62805" + }, "osbuild": { "commit": "8d2a28c5b984667d29aaeca101b5a2b1d72605e6" } @@ -100,6 +127,9 @@ }, "rhel-9.6": { "dependencies": { + "composer": { + "commit": "73e71e29381cd96fe4666091714d0b8c90a62805" + }, "osbuild": { "commit": "8d2a28c5b984667d29aaeca101b5a2b1d72605e6" } @@ -107,6 +137,9 @@ }, "rhel-9.7": { "dependencies": { + "composer": { + "commit": "73e71e29381cd96fe4666091714d0b8c90a62805" + }, "osbuild": { "commit": "8d2a28c5b984667d29aaeca101b5a2b1d72605e6" } @@ -153,6 +186,9 @@ }, "rhel-9.8": { "dependencies": { + "composer": { + "commit": "73e71e29381cd96fe4666091714d0b8c90a62805" + }, "osbuild": { "commit": "8d2a28c5b984667d29aaeca101b5a2b1d72605e6" } @@ -199,6 +235,9 @@ }, "rhel-10.0": { "dependencies": { + "composer": { + "commit": "73e71e29381cd96fe4666091714d0b8c90a62805" + }, "osbuild": { "commit": "8d2a28c5b984667d29aaeca101b5a2b1d72605e6" } @@ -206,6 +245,9 @@ }, "rhel-10.1": { "dependencies": { + "composer": { + "commit": "73e71e29381cd96fe4666091714d0b8c90a62805" + }, "osbuild": { "commit": "8d2a28c5b984667d29aaeca101b5a2b1d72605e6" } @@ -213,6 +255,9 @@ }, "rhel-10.2": { "dependencies": { + "composer": { + "commit": "73e71e29381cd96fe4666091714d0b8c90a62805" + }, "osbuild": { "commit": "8d2a28c5b984667d29aaeca101b5a2b1d72605e6" } @@ -259,6 +304,9 @@ }, "centos-9": { "dependencies": { + "composer": { + "commit": "73e71e29381cd96fe4666091714d0b8c90a62805" + }, "osbuild": { "commit": "8d2a28c5b984667d29aaeca101b5a2b1d72605e6" } @@ -266,6 +314,9 @@ }, "centos-stream-9": { "dependencies": { + "composer": { + "commit": "73e71e29381cd96fe4666091714d0b8c90a62805" + }, "osbuild": { "commit": "8d2a28c5b984667d29aaeca101b5a2b1d72605e6" } @@ -312,6 +363,9 @@ }, "centos-10": { "dependencies": { + "composer": { + "commit": "73e71e29381cd96fe4666091714d0b8c90a62805" + }, "osbuild": { "commit": "8d2a28c5b984667d29aaeca101b5a2b1d72605e6" } @@ -319,6 +373,9 @@ }, "centos-stream-10": { "dependencies": { + "composer": { + "commit": "73e71e29381cd96fe4666091714d0b8c90a62805" + }, "osbuild": { "commit": "8d2a28c5b984667d29aaeca101b5a2b1d72605e6" } diff --git a/schutzbot/deploy.sh b/schutzbot/deploy.sh index dfc1b7cc..369d1b48 100755 --- a/schutzbot/deploy.sh +++ b/schutzbot/deploy.sh @@ -56,101 +56,6 @@ priority=${priority} EOF } -function get_last_passed_commit { - # Using 'internal' instead of 'true' so it's easier to see the pipelines in the Gitlab page - if [ "${INTERNAL_NIGHTLY:=false}" == "internal" ]; then - project_id="34771166" - - # To get the schedule id use the ../pipeline_schedule endpoint - if [[ ${VERSION_ID%.*} == "9" ]]; then - # RHEL 9 scheduled pipeline id - schedule_id="233736" - elif [[ ${VERSION_ID%.*} == "10" ]]; then - # RHEL 10 scheduled pipeline id (FYI - it was used for RHEL 8 before) - schedule_id="233735" - else - echo "No scheduled pipeline defined for RHEL $VERSION_ID" - exit 1 - fi - - # Last executed pipeline ID - schedule_info=$(curl -s --header "PRIVATE-TOKEN: ${GITLAB_API_TOKEN}" "https://gitlab.com/api/v4/projects/${project_id}/pipeline_schedules/${schedule_id}") - - # Check if API returned an error (like 401 Unauthorized) - if echo "$schedule_info" | jq -e '.message' >/dev/null; then - echo "GitLab API Error: $(echo "$schedule_info" | jq -r .message)" - exit 1 - fi - - pipeline_id=$(echo "$schedule_info" | jq -r '.last_pipeline.id // empty') - - # Ensure pipeline_id is not null or empty before proceeding - if [[ -z "$pipeline_id" || "$pipeline_id" == "null" ]]; then - echo "Error: Could not find the last pipeline ID for schedule ${schedule_id}." - exit 1 - fi - - number_of_days=7 - warning_date=$(date -d "- $number_of_days days" +%s) - - pipeline_info=$(curl -s --header "PRIVATE-TOKEN: ${GITLAB_API_TOKEN}" "https://gitlab.com/api/v4/projects/${project_id}/pipelines/${pipeline_id}") - created_at=$(echo "$pipeline_info" | jq -r '.started_at // empty') - - if [[ -z "$created_at" || "$created_at" == "null" ]]; then - echo "Error: Could not determine start time for pipeline ${pipeline_id}." - exit 1 - fi - - if [[ $(date -d "${created_at}" +%s) -lt "${warning_date}" ]]; then - echo "We are using an old scheduled pipeline id (more than $number_of_days days ago). Please update it" - exit 1 - fi - - statuses=$(curl -s --header "PRIVATE-TOKEN: ${GITLAB_API_TOKEN}" "https://gitlab.com/api/v4/projects/${project_id}/pipelines/${pipeline_id}/jobs?per_page=100" | jq -cr '.[] | select(.stage=="rpmbuild") | .status') - for status in ${statuses}; do - if [ "$status" == "failed" ]; then - echo "Last nightly pipeline ('rpmbuild' stage) failed in osbuild-composer CI. We will not run nightly-internal jobs in CIV." - exit 1 - fi - done - - commit=$(echo "$pipeline_info" | jq -r '.sha') - echo "$commit" - - else - # Capture response and HTTP code to handle GitHub API failures (e.g. 401, 403) - response=$(curl -u "${API_USER}:${API_PAT}" -s -w "%{http_code}" "https://api.github.com/repos/osbuild/osbuild-composer/commits?per_page=100") - http_code="${response: -3}" - body="${response::-3}" - - if [ "$http_code" != "200" ]; then - echo "GitHub API Error (HTTP $http_code): $body" - exit 1 - fi - - commit_list=$(echo "$body" | jq -cr '.[].sha') - - # Initialize final_commit to prevent "unbound variable" error if no commit matches - final_commit="" - - for commit_sha in ${commit_list}; do - gitlab_status=$(curl -u "${API_USER}:${API_PAT}" -s "https://api.github.com/repos/osbuild/osbuild-composer/commits/${commit_sha}/status" \ - | jq -cr '.statuses[] | select(.context == "Schutzbot on GitLab") | .state') - if [[ ${gitlab_status} == "success" ]]; then - final_commit=$commit_sha - break - fi - done - - if [[ -z "$final_commit" ]]; then - echo "Error: No successful commits found in the last 100 entries." - exit 1 - fi - - echo "$final_commit" - fi -} - # Get OS details. source ci/set-env-variables.sh @@ -188,10 +93,9 @@ echo -e "fastestmirror=1" | sudo tee -a /etc/dnf/dnf.conf # TODO: include this in the jenkins runner (and split test/target machines out) sudo dnf -y install jq -# Get latest commit from osbuild-composer main branch -GIT_COMMIT=$(get_last_passed_commit) - -setup_repo osbuild-composer "${GIT_COMMIT}" 5 +# Get pinned commit for osbuild-composer to install RPMs +COMPOSER_GIT_COMMIT=$(jq -r '.["'"${ID}-${VERSION_ID}"'"].dependencies.composer.commit' Schutzfile) +setup_repo osbuild-composer "${COMPOSER_GIT_COMMIT}" 5 OSBUILD_GIT_COMMIT=$(cat Schutzfile | jq -r '.["'"${ID}-${VERSION_ID}"'"].dependencies.osbuild.commit') if [[ "${OSBUILD_GIT_COMMIT}" != "null" ]]; then diff --git a/schutzbot/update-schutzfile-composer b/schutzbot/update-schutzfile-composer new file mode 100755 index 00000000..3568e2fa --- /dev/null +++ b/schutzbot/update-schutzfile-composer @@ -0,0 +1,72 @@ +#!/usr/bin/env python3 +import json +import os +import sys +import urllib.request + +SCHUTZFILE = "Schutzfile" + + +def composer_main_commit_id(): + token = os.environ.get("GH_TOKEN") + req = urllib.request.Request("https://api.github.com/repos/osbuild/osbuild-composer/commits/main") + req.add_header("Accept", "application/vnd.github+json") + if token: + # this API request doesn't necessarily require a token, but let's use it if we have one + req.add_header("Authorization", f"Bearer {token}") + try: + with urllib.request.urlopen(req, timeout=30) as resp: + body = resp.read() + except urllib.error.HTTPError as http_error: + print(http_error) + sys.exit(1) + + data = json.loads(body) + if "[skip ci]" in data["commit"]["message"]: + # Post-release version bump commits contain the message [skip ci] which skips running CI. These commits do not + # have RPMs associated with them, so they shouldn't be used to downstream testing. Use the parent commit ID if + # we detect [skip ci] in the commit message on main. + # Our commit history is linear, so we can safely assume that there is only one parent. + # Also, there should never be two [skip ci] commits in a row, so we don't need to verify the commit message of + # the parent. + print("[skip ci] found in main commit message. Using parent.") + return data["parents"][0]["sha"] + + return data["sha"] + + +def update_composer_commit_ids(new): + with open(SCHUTZFILE, encoding="utf-8") as schutzfile: + data = json.load(schutzfile) + + unique_changes = set() + + for distro in data.keys(): + if distro == "common": + continue + + old = data[distro].get("dependencies", {}).get("composer", {}).get("commit") + if old: + change = f"Changes: https://github.com/osbuild/osbuild-composer/compare/{old}...{new}" + if change not in unique_changes: + unique_changes.add(change) + data[distro].setdefault("dependencies", {}).setdefault("composer", {})["commit"] = new + + with open(SCHUTZFILE, encoding="utf-8", mode="w") as schutzfile: + json.dump(data, schutzfile, indent=" ") + + with open("github_pr_body.txt", encoding="utf-8", mode="w") as pr_body: + pr_body.write("Updating osbuild-composer dependency commit IDs to current `main`\n\n") + pr_body.write("\n".join(unique_changes)) + pr_body.write("\n") + + +def main(): + main_id = composer_main_commit_id() + print(f"osbuild/osbuild-composer main commit ID: {main_id}") + print("Updating Schutzfile") + update_composer_commit_ids(main_id) + + +if __name__ == "__main__": + main()