ci: Adding S3 storage

Author: psaavedra

.github/actions/bitbake-build/action.yml

@@ -1,6 +1,10 @@
 inputs:
+  bitbake_buildname:
+    default: 'build'
   bitbake_prefix:
     default: ''
+  bitbake_machine:
+    default: 'raspberrypi5'
   bitbake_target:
     default: 'core-image-weston-wpe'
   bitbake_source:
@@ -15,6 +19,8 @@ runs:
       run:  |
         echo "Environment:"
         echo "======================================================================"
+        echo "BITBAKE_BUILDNAME=${{ inputs.bitbake_buildname }}"
+        echo "BITBAKE_MACHINE=${{ inputs.bitbake_machine }}"
         echo "BITBAKE_PREFIX=${{ inputs.bitbake_prefix }}"
         echo "BITBAKE_SOURCE=${{ inputs.bitbake_source }}"
         echo "BITBAKE_TARGET=${{ inputs.bitbake_target }}"
@@ -24,18 +30,39 @@ runs:
       shell: bash
       run:  |
         cd ~/yocto-meta-wpe-image-${{ inputs.repo_release }}
-        rm -rf tmp
         ulimit -n 4096
-        #source sources/meta-wpe-image/scripts/setup-environment ${{ inputs.bitbake_source }}
-        source setup-environment ${{ inputs.bitbake_source }}
+        source sources/meta-wpe-image/scripts/setup-environment ${{ inputs.bitbake_buildname}} ${{ inputs.bitbake_machine }} ${{ inputs.bitbake_source }}
+        MAX_RETRIES=5
+        export ${{ inputs.bitbake_prefix }}
+        for i in $(seq 1 $MAX_RETRIES); do
+            echo "Attempt $i to fetch sources for $IMAGE..."
+            if bitbake ${{ inputs.bitbake_target }} --runall=fetch; then
+                echo "All sources fetched successfully."
+                break
+            fi
+
+            if [ $i -lt $MAX_RETRIES ]; then
+                echo "Retrying fetch process..."
+            else
+                echo "Max retries reached. Some sources may still be missing."
+            fi
+        done
         ${{ inputs.bitbake_prefix }} bitbake ${{ inputs.bitbake_target }}
+    - name: Generate timestamped prefix
+      id: timestamp
+      shell: bash
+      run: |
+        TIMESTAMP=$(date +"%Y%m%d%H%M")
+        echo "prefix=${TIMESTAMP}" >> $GITHUB_OUTPUT
     - name: Artifacts
       if: always()
       shell: bash
-      run:  |
-        ls -lh ~/yocto-meta-wpe-image-${{ inputs.repo_release }}/builds/*/tmp/deploy/images/*/
+      run: |
+        ls -lh ~/yocto-meta-wpe-image-${{ inputs.repo_release }}/builds/${{ inputs.bitbake_buildname }}/tmp/deploy/images/${{ inputs.bitbake_machine }}/
+        PREFIX=${{ steps.timestamp.outputs.prefix }}
+        s3cmd put --recursive ~/yocto-meta-wpe-image-${{ inputs.repo_release }}/builds/${{ inputs.bitbake_buildname }}/tmp/deploy/images/${{ inputs.bitbake_machine }}  s3://yocto/${PREFIX}-${{ inputs.bitbake_buildname }}-${{ inputs.bitbake_target }}/
     - name: Clean the tmp dir
       if: always()
       shell: bash
       run:  |
-        rm -rf ~/yocto-meta-wpe-image-${{ inputs.repo_release }}/builds/*/tmp
+        rm -rf ~/yocto-meta-wpe-image-${{ inputs.repo_release }}/builds/${{ inputs.bitbake_buildname }}/tmp

.github/actions/s3-configure/action.yml

@@ -0,0 +1,28 @@
+inputs:
+  access_key:
+    description: "S3 Access Key"
+    required: true
+  secret_key:
+    description: "S3 Secret Key"
+    required: true
+  host_base:
+    description: "S3 Host base"
+    required: false
+    default: "ams3.digitaloceanspaces.com"
+
+runs:
+  using: "composite"
+  steps:
+    - name: Configure s3cmd
+      shell: bash
+      run: |
+        cat > ~/.s3cfg <<EOL
+        [default]
+        access_key = ${{ inputs.access_key }}
+        secret_key = ${{ inputs.secret_key }}
+        host_base = ${{ inputs.host_base }}
+        host_bucket = %(bucket)s.${{ inputs.host_base }}
+        use_https = True
+        gpg_command = /usr/bin/gpg
+        EOL
+

.github/workflows/test-build.yml

@@ -5,7 +5,24 @@ jobs:
     runs-on: [self-hosted, builder]
     if: "!contains(github.event.head_commit.message, 'ci skip')"
     steps:
+      - name: test
+        env:
+          S3_ACCESS_KEY: ${{ secrets.S3_ACCESS_KEY }}
+          S3_SECRET_KEY: ${{ secrets.S3_SECRET_KEY }}
+          S3_HOST_BASE: ${{ secrets.S3_HOST_BASE }}
+        run: |
+          echo secrets.S3_HOST_BASE
+          echo ${S3_HOST_BASE}
       - uses: actions/checkout@v3
+      - uses: ./.github/actions/s3-configure
+        env:
+          S3_ACCESS_KEY: ${{ secrets.S3_ACCESS_KEY }}
+          S3_SECRET_KEY: ${{ secrets.S3_SECRET_KEY }}
+          S3_HOST_BASE: ${{ secrets.S3_HOST_BASE }}
+        with:
+          access_key: ${S3_ACCESS_KEY}
+          secret_key: ${S3_SECRET_KEY}
+          host_base: ${S3_HOST_BASE}
       - uses: ./.github/actions/bitbake-repo
         with:
           repo_release: 'scarthgap'
@@ -25,7 +42,9 @@ jobs:
       #- uses: actions/checkout@v3
       - uses: ./.github/actions/bitbake-build
         with:
-          bitbake_source: 'raspberrypi3-mesa-wpe-${{ matrix.wpe_vers }} ${{ matrix.machine }} poky-wayland layers.raspberrypi conf.wpe-${{ matrix.wpe_vers }} --update-config'
+          bitbake_buildname: wpe-${{ matrix.wpe_vers }}-${{ matrix.machine }}
+          bitbake_machine: ${{ matrix.machine }}
+          bitbake_source: 'poky-wayland layers.raspberrypi conf.wpe-${{ matrix.wpe_vers }} --update-config'
           repo_release: ${{ matrix.yocto_rel }}
 
   # scarthgap-raspberrypi3-mesa-weston-wpe-nightly: