Adding multi-arch multi-images implementation (#33)

* adding the ubi9 builder and moving all of them under the buildsers directory

* Adding builders.json file to use on the workflows

* refactor: fixing the push image workflow to iterate over the builders

* feat: adding a publish.sh script publishing builder easier

* refactor: Updating create release workflow to iterate over the builders

* fix: moving options.json to the right location

* fix: smoke.sh tests

* fix: create-release workflow

* fix: chaging permissions for the publish.sh script

fix: updating publish.sh script

* fix: relase notes

fix: release notes path

* fix: setting the correct url for uploading artifacts

* fix: create release workflow

* fix: push image workflow

fix: installing crane on push image workflow

* fix: push image workflow

* fix: ceate relase workflow

* doc: Updating readme file with the ubi 9 builder

* fix: update-builder workflow

* fix: adding files on .syncignore of the scripts

* fix: lint failures

Author: pacostas

.github/workflows/create-release.yml

@@ -7,10 +7,94 @@ on:
 
 concurrency: release
 
+env:
+  BUILDERS_FILEPATH: "builders.json"
+
 jobs:
+  preparation:
+    name: Preparation
+    runs-on: ubuntu-24.04
+    outputs:
+      builders: ${{ steps.get-builders.outputs.builders }}
+    steps:
+    - name: Checkout
+      uses: actions/checkout@v4
+
+    - name: Get Builders
+      id: get-builders
+      run: |
+        builders=$(jq -n -c '[]')
+
+        if [ -f ${{ env.BUILDERS_FILEPATH }} ]; then
+          builders=$(jq -c '.builders' ${{ env.BUILDERS_FILEPATH }})
+        else
+          # Strip off the Github org prefix from repo name
+          # paketo-buildpacks/builder-with-some-name --> builder-with-some-name
+          registry_repo=$(echo "${{ github.repository }}" | sed 's/^.*\///')
+          builders=$(jq -n -c '[
+              {
+                "name": "default",
+                "path": ".",
+                "container_repository": "'"${registry_repo}"'"
+              }
+            ]')
+        fi
+
+        # Filter only the necessary fields
+        builders=$(echo "$builders" | jq 'map({
+          name,
+          path,
+          container_repository
+        })')
+
+        builders=$(jq -c <<< "$builders")
+        echo "builders=$builders"
+        echo "builders=$builders" >> "$GITHUB_OUTPUT"
+
+  builder_files_changed:
+    name: Builder Files Changed
+    runs-on: ubuntu-24.04
+    needs: preparation
+    outputs:
+      builders_changed: ${{ steps.compare_previous_release.outputs.builders_changed }}
+    steps:
+    - name: Checkout
+      uses: actions/checkout@v4
+      with:
+        fetch-depth: 0  # gets full history
+
+    - name: Compare previous releases
+      id: compare_previous_release
+      run: |
+        builders=$(echo '${{ needs.preparation.outputs.builders }}' | jq -c '.[]')
+        for builder in $builders; do
+          builder_path=$(echo "$builder" | jq -r '.path')
+
+          changed=$(git diff --name-only $(git describe --tags --abbrev=0) -- $builder_path/builder.toml)
+          if [ -z "${changed}" ]
+          then
+            # We do not write on the github output because this step runs multiple times
+            # and it might overwrite any true value written by the else statement.
+            echo "No changes for $builder_path/builder.toml"
+          else
+            echo "Changes detected for $builder_path/builder.toml"
+            echo "builders_changed=true" >> $GITHUB_OUTPUT
+          fi
+        done
+
   smoke:
     name: Smoke Test
     runs-on: ubuntu-24.04
+    needs: [preparation, builder_files_changed]
+    if: ${{ needs.builder_files_changed.outputs.builders_changed == 'true' }}
+    strategy:
+      matrix:
+        builders: ${{ fromJSON(needs.preparation.outputs.builders) }}
+    services:
+      registry:
+        image: registry:2
+        ports:
+          - 5000:5000
     outputs:
       release_notes: ${{ steps.notes.outputs.body }}
     steps:
@@ -34,43 +118,91 @@ jobs:
         pack-version: ${{ steps.pack-version.outputs.version }}
 
     - name: Run Smoke Tests
-      run: ./scripts/smoke.sh --name builder
+      run: |
+        ./scripts/smoke.sh --builder-dir "${{ matrix.builders.path }}"
 
-    - name: Generate Release Notes
+    - name: Generate builder information
       id: notes
       run: |
-        notes="$(pack inspect-builder builder | grep -v 'Inspecting builder' \
+        registry_builder_name=localhost:5000/${{ matrix.builders.name }}
+
+        ./scripts/publish.sh --builder-toml-path "${{ matrix.builders.path }}/builder.toml" \
+          --builder-image-ref "${registry_builder_name}"
+
+        pack inspect-builder $registry_builder_name | grep -v 'Inspecting builder' \
           | grep -v 'REMOTE:' \
           | grep -v 'LOCAL:' \
           | grep -v '\(not present\)' \
-          | grep -v 'Warning' \
-          | sed -e '/./,$!d' \
-          | awk -F, '{printf "%s\\n", $0}')"
-        echo "body=${notes}" >> "$GITHUB_OUTPUT"
+          | grep -v 'Warning' > "${{ matrix.builders.name }}-information.md"
+
+    - name: Upload information for ${{ matrix.builders.name }}
+      uses: actions/upload-artifact@v4
+      with:
+        name: "${{ matrix.builders.name }}-information.md"
+        path: "${{ matrix.builders.name }}-information.md"
 
   release:
     name: Release
     runs-on: ubuntu-24.04
-    needs: smoke
+    needs: [smoke, preparation]
     steps:
     - name: Checkout With History
       uses: actions/checkout@v4
       with:
         fetch-depth: 0  # gets full history
 
-    - name: Compare With Previous Release
-      id: compare_previous_release
+    - name: Download Release Note File(s)
+      uses: actions/download-artifact@v4
+      with:
+        path: information
+        pattern: "*information.md"
+        merge-multiple: true
+
+    - name: Generate Release Notes and Assets
+      id: generate_release_notes_and_assets
       run: |
-        if [ -z "$(git diff $(git describe --tags --abbrev=0) -- builder-buildpackless.toml)" ]
-        then
-          echo "builder_changes=false" >> "$GITHUB_OUTPUT"
-        else
-          echo "builder_changes=true" >> "$GITHUB_OUTPUT"
-        fi
+        set -euo pipefail
+        shopt -s inherit_errexit
+
+        builders=$(echo '${{ needs.preparation.outputs.builders }}' | jq -c '.[]')
+        release_notes_dir="information"
+
+        # Start with an empty array
+        assets=$(jq -n -c '[]')
+
+        # Generate release assets
+        for builder in $builders; do
+          builder_name=$(echo "$builder" | jq -r '.name')
+
+          assets="$(jq -c \
+            --arg release_notes_dir "${release_notes_dir}" \
+            --arg builder_name "${builder_name}" \
+            '. += [
+              {
+                "path": ($release_notes_dir + "/" + $builder_name + "-" + "information.md"),
+                "name": ($builder_name + "-" + "information.md"),
+                "content_type": "text/markdown"
+              }
+            ]' <<<"${assets}")"
+        done
+
+        echo "assets=${assets}" >> "$GITHUB_OUTPUT"
+
+        # translates 'paketo-buildpacks' to 'paketobuildpacks'
+        DOCKERHUB_ORG="${GITHUB_REPOSITORY_OWNER/-/}"
+
+        # Generate release notes
+        release_body=""
+        for builder in $builders; do
+          container_repository=$(echo "$builder" | jq -r '.container_repository')
+          release_body+="Builder: \`${DOCKERHUB_ORG}/${container_repository}\`\n"
+        done
+
+        payload="{\"body\" : \"\n${release_body}\"}"
+        echo "release_body=${payload}" >> "$GITHUB_OUTPUT"
 
     - name: Publish Release
       id: publish
-      if: ${{ steps.compare_previous_release.outputs.builder_changes == 'true' }}
       uses: release-drafter/release-drafter@v6
       with:
         config-name: release-drafter-config.yml
@@ -79,22 +211,38 @@ jobs:
         GITHUB_TOKEN: ${{ secrets.PAKETO_BOT_GITHUB_TOKEN }}
 
     - name: Update Release Notes
-      if: ${{ steps.compare_previous_release.outputs.builder_changes == 'true' }}
       run: |
         set -euo pipefail
         shopt -s inherit_errexit
 
-        payload="{\"body\" : \"\`\`\`\n${RELEASE_BODY}\n\`\`\`\"}"
-
+        payload='${{ steps.generate_release_notes_and_assets.outputs.release_body }}'
         curl --fail \
           -X PATCH \
           -H "Accept: application/vnd.github.v3+json" \
           -H "Authorization: token ${GITHUB_TOKEN}" \
           "https://api.github.com/repos/${GITHUB_REPOSITORY}/releases/${RELEASE_ID}" \
           -d "${payload}"
+
+        assets='${{ steps.generate_release_notes_and_assets.outputs.assets }}'
+
+        # Update release assets
+        for row in $(echo "${assets}" | jq -c '.[]'); do
+          path=$(echo "$row" | jq -r '.path')
+          name=$(echo "$row" | jq -r '.name')
+          content_type=$(echo "$row" | jq -r '.content_type')
+
+          echo "Uploading asset: ${name} from path: ${path}"
+
+          curl -L \
+            -X POST \
+            -H "Accept: application/vnd.github.v3+json" \
+            -H "Authorization: token ${GITHUB_TOKEN}" \
+            -H "Content-Type: ${content_type}" \
+            "https://uploads.github.com/repos/${GITHUB_REPOSITORY}/releases/${RELEASE_ID}/assets?name=${name}" \
+            --data-binary "@${path}"
+        done
       env:
         RELEASE_ID: ${{ steps.publish.outputs.id }}
-        RELEASE_BODY: ${{ needs.smoke.outputs.release_notes }}
         GITHUB_TOKEN: ${{ secrets.PAKETO_BOT_GITHUB_TOKEN }}
 
   failure:

.github/workflows/push-image.yml

@@ -5,60 +5,90 @@ on:
     types:
     - published
 
+env:
+  BUILDERS_FILEPATH: "builders.json"
+
 jobs:
-  push:
-    name: Push
+  preparation:
+    name: Preparation
     runs-on: ubuntu-24.04
+    outputs:
+      builders: ${{ steps.get-builders.outputs.builders }}
+      tag: ${{ steps.event.outputs.tag }}
+      pack_version: ${{ steps.pack-version.outputs.version }}
     steps:
-
-    - name: Parse Event
-      id: event
-      run: |
-        echo "tag=$(jq -r '.release.tag_name' "${GITHUB_EVENT_PATH}" | sed s/^v//)" >> "$GITHUB_OUTPUT"
-
     - name: Checkout
       uses: actions/checkout@v4
 
-    - name: Get pack version
-      id: pack-version
+      # TODO: Add the image re on the buiders.json default
+    - name: Get Builders
+      id: get-builders
       run: |
-        version=$(jq -r .pack "scripts/.util/tools.json")
-        echo "version=${version#v}" >> "$GITHUB_OUTPUT"
+        builders=$(jq -n -c '[]')
 
-    - name: Install Global Pack
-      uses: buildpacks/github-actions/setup-pack@main
-      with:
-        pack-version: ${{ steps.pack-version.outputs.version }}
+        if [ -f ${{ env.BUILDERS_FILEPATH }} ]; then
+          builders=$(jq -c '.builders' ${{ env.BUILDERS_FILEPATH }})
+        else
+          # Strip off the Github org prefix from repo name
+          # paketo-buildpacks/builder-with-some-name --> builder-with-some-name
+          registry_repo=$(echo "${{ github.repository }}" | sed 's/^.*\///')
+          builders=$(jq -n -c '[
+              {
+                "name": "default",
+                "path": ".",
+                "container_repository": "'"${registry_repo}"'"
+              }
+            ]')
+        fi
 
+        # Filter only the necessary fields
+        builders=$(echo "$builders" | jq 'map({
+          name,
+          path,
+          container_repository
+        })')
 
-    - name: Install crane
+        builders=$(jq -c <<< "$builders")
+        echo "builders=$builders"
+        echo "builders=$builders" >> "$GITHUB_OUTPUT"
+
+    - name: Parse Event
+      id: event
       run: |
-        VERSION=$(curl -s "https://api.github.com/repos/google/go-containerregistry/releases/latest" | jq -r '.tag_name')
-        OS=Linux
-        ARCH=x86_64
-        curl -sL "https://github.com/google/go-containerregistry/releases/download/${VERSION}/go-containerregistry_${OS}_${ARCH}.tar.gz" > go-containerregistry.tar.gz
+        echo "tag=$(jq -r '.release.tag_name' "${GITHUB_EVENT_PATH}" | sed s/^v//)" >> "$GITHUB_OUTPUT"
 
-        tar -zxvf go-containerregistry.tar.gz -C "${HOME}/bin" crane
-        chmod +x "${HOME}/bin/crane"
+  push:
+    name: Push
+    needs: preparation
+    runs-on: ubuntu-24.04
+    strategy:
+      matrix:
+        builders: ${{ fromJSON(needs.preparation.outputs.builders) }}
+    steps:
 
-    - name: Enable Experimental Pack Features
-      run: |
-        if [ -f "scripts/options.json" ] && jq -e -r .pack_config_enable_experimental "scripts/options.json" > /dev/null; then
-          pack config experimental true
-        fi
+    - name: Checkout
+      uses: actions/checkout@v4
 
     - name: Create Builder Image and Push To Dockerhub
       env:
         PAKETO_BUILDPACKS_DOCKERHUB_USERNAME: ${{ secrets.PAKETO_BUILDPACKS_DOCKERHUB_USERNAME }}
         PAKETO_BUILDPACKS_DOCKERHUB_PASSWORD: ${{ secrets.PAKETO_BUILDPACKS_DOCKERHUB_PASSWORD }}
         GITHUB_REPOSITORY_OWNER: ${{ github.repository_owner }}
       run: |
+        # shellcheck source=./scripts/.util/tools.sh
+        source "./scripts/.util/tools.sh"
+
+        util::tools::crane::install --directory "./.bin"
+
         DOCKERHUB_ORG="${GITHUB_REPOSITORY_OWNER/-/}" # translates 'paketo-buildpacks' to 'paketobuildpacks'
-        registry_repo="ubi-9-builder-buildpackless"
+        container_repository=${{ matrix.builders.container_repository }}
 
         echo "${PAKETO_BUILDPACKS_DOCKERHUB_PASSWORD}" | docker login --username "${PAKETO_BUILDPACKS_DOCKERHUB_USERNAME}" --password-stdin
-        pack builder create "${DOCKERHUB_ORG}/${registry_repo}:${{ steps.event.outputs.tag }}" --config builder-buildpackless.toml --publish
-        crane copy "${DOCKERHUB_ORG}/${registry_repo}:${{ steps.event.outputs.tag }}" "${DOCKERHUB_ORG}/${registry_repo}:latest"
+
+        ./scripts/publish.sh --builder-toml-path "${{ matrix.builders.path }}/builder.toml" \
+          --builder-image-ref "${DOCKERHUB_ORG}/${container_repository}:${{ needs.preparation.outputs.tag }}"
+
+        ./.bin/crane copy "${DOCKERHUB_ORG}/${container_repository}:${{ needs.preparation.outputs.tag }}" "${DOCKERHUB_ORG}/${container_repository}:latest"
 
   failure:
     name: Alert on Failure