QML Pipeline Improvements (#1322)

Co-authored-by: anthayes92 <[email protected]>
Co-authored-by: Alan Martin <[email protected]>
Co-authored-by: Alan Martin <[email protected]>
Co-authored-by: Paul Finlay <[email protected]>

Author: 4 people

.github/workflows/v2-build-and-deploy.yml

@@ -0,0 +1,101 @@
+name: V2 Build and Deploy Demos
+
+permissions:
+  contents: read
+  pull-requests: write
+  actions: read
+
+on:
+  workflow_dispatch:
+    inputs:
+      environment:
+        description: SWC environment to deploy to.
+        options:
+          - swc-staging
+          - swc-prod
+          - swc-dev
+        required: true
+        type: choice
+      target:
+        default: stable
+        description: PennyLane version to build the demos. Either 'latest' or the most recent 'stable' release.
+        options:
+          - latest
+          - stable
+        required: true
+        type: choice
+      demos:
+        description: Demos to build and deploy, space-separated list of slugs (e.g. "demo1 demo2 demo3"), or leave empty for all demos.
+        required: false
+        type: string
+      as-previews:
+        default: false
+        description: |
+          Whether to deploy the demos as previews.
+
+          **Please note** that demos built with the latest version cannot be published to swc-staging or swc-prod.
+          They can only be deployed as previews.
+        required: false
+        type: boolean
+
+jobs:
+  validate-and-parse-inputs:
+    runs-on: ubuntu-latest
+    outputs:
+      branch: ${{ steps.set-branch.outputs.branch }}
+    steps:
+      - name: Set branch
+        id: set-branch
+        run: |
+          if [[ "${{ github.event.inputs.target }}" == "stable" ]]; then
+            echo "branch=master" >> $GITHUB_OUTPUT
+          elif [[ "${{ github.event.inputs.target }}" == "latest" ]]; then
+            echo "branch=dev" >> $GITHUB_OUTPUT
+          else 
+            echo "branch=" >> $GITHUB_OUTPUT
+          fi
+
+      - name: Validate preview input
+        id: validate-preview
+        run: |
+          if [[ 
+            ("${{ github.event.inputs.environment }}" == "swc-staging" || 
+            "${{ github.event.inputs.environment }}" == "swc-prod") && 
+            "${{ github.event.inputs.target }}" == "latest" && 
+            "${{ github.event.inputs.as-previews }}" == "false" 
+          ]]; then
+            echo "========================="
+            echo "🚫 Invalid input detected:"
+            echo "Demos built with the latest version cannot be published to 'swc-staging' or 'swc-prod'."
+            echo "They can only be deployed as previews."
+            echo "Please set the 'as-previews' input to 'true' in your workflow configuration."
+            echo "========================="
+            exit 1
+          fi
+
+  build:
+    needs: validate-and-parse-inputs
+    if: >
+      (needs.validate-and-parse-inputs.outputs.branch == 'master') ||
+      (needs.validate-and-parse-inputs.outputs.branch == 'dev')
+    uses: ./.github/workflows/v2-build-demos.yml
+    with:
+      ref: ${{ needs.validate-and-parse-inputs.outputs.branch }}
+      demo-names: ${{ github.event.inputs.demos }}
+      dev: ${{ github.event.inputs.target == 'latest' }}
+      save-artifact: true
+      artifact-name: build-and-deploy-${{ github.event.inputs.target }}
+      keep-going: false
+      quiet: false
+      batch_size: 10
+
+  deploy:
+    uses: ./.github/workflows/v2-deploy-demos.yml
+    needs:
+      - validate-and-parse-inputs
+      - build
+    secrets: inherit
+    with:
+      environment: ${{ github.event.inputs.environment }}
+      artifact-name: build-and-deploy-${{ github.event.inputs.target }}
+      preview: ${{ github.event.inputs.as-previews }}

.github/workflows/v2-build-demos.yml

@@ -8,13 +8,20 @@ on:
         type: string
       demo-names:
         description: |
-          Instead of building all demos, only build the demos specified in this list.
+          Only build the demos specified in a space-separated list,
+          e.g. demo1 demo2 demo3.
+
+          If not specified, all demos will be built.
         required: false
         type: string
         default: ''
+      dev:
+        description: Use development dependencies.
+        required: false
+        type: boolean
+        default: false
       save-artifact:
-        description: |
-          Whether to save the built demos as an artifact.
+        description: Save the built demos as an artifact.
         required: false
         type: boolean
         default: true
@@ -24,22 +31,19 @@ on:
         type: string
         default: ''
       keep-going:
-        description: Whether to keep going if a demo fails to build
+        description: Keep going if a demo fails to build.
         required: false
         type: boolean
         default: false
       quiet:
-        description: |
-          Whether to suppress output from the build process
+        description: Suppress output from the build process.
         required: false
         type: boolean
         default: false
       batch_size:
-        description: |
-          Number of demos to build per job
+        description: Number of demos to build per job.
         type: number
         default: 10
-
     outputs:
       artifact-name:
         description: "Name of the artifact containing the built demos"
@@ -57,25 +61,28 @@ on:
         required: false
         type: string
         default: ''
+      dev:
+          description: Use development dependencies.
+          required: false
+          type: boolean
+          default: false
       save-artifact:
-        description: |
-          Whether to save the built demos as an artifact.
+        description: Save the built demos as an artifact.
         required: false
         type: boolean
         default: true
       artifact-name:
-        description: Name of the artifact containing the built demos (defaults to demo-build-{ref})
-        required: true
+        description: Name of the build artifact (defaults to demo-build-{ref}).
+        required: false
         type: string
         default: ''
       keep-going:
-        description: Whether to keep going if a demo fails to build
+        description: Keep going if a demo fails to build.
         required: false
         type: boolean
         default: false
       quiet:
-        description: |
-          Whether to suppress output from the build process
+        description: Suppress output from the build process.
         required: false
         type: boolean
         default: false
@@ -165,7 +172,8 @@ jobs:
             --execute \
             ${{ inputs.keep-going && '--keep-going' || '--no-keep-going' }} \
             ${{ inputs.quiet && '--quiet' || '--no-quiet' }} \
-            ${{ steps.demo-name-list.outputs.demo-names }}
+            ${{ steps.demo-name-list.outputs.demo-names }} \
+            ${{ inputs.dev && '--dev' || '--no-dev' }}
       
       - name: Upload artifacts
         id: upload-artifacts
@@ -207,5 +215,3 @@ jobs:
         uses: geekyeggo/delete-artifact@v5
         with:
             name: ${{ needs.generate-build-variables.outputs.artifact-name }}-c*
-
-  

.github/workflows/v2-build-pr.yml

@@ -0,0 +1,81 @@
+name: V2 Build PR
+
+on:
+  pull_request:
+    branches: [master, dev]
+
+permissions:
+  contents: read
+
+concurrency:
+  group: build-v2-demos-${{ github.event.pull_request.head.sha }}
+  cancel-in-progress: true
+
+jobs:
+  # Step 1: Identify changed demos
+  identify-changed-demos:
+    runs-on: ubuntu-latest
+    outputs:
+      updated: ${{ steps.get-changed-demos.outputs.updated }}
+      deleted: ${{ steps.get-changed-demos.outputs.deleted }}
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Get changed demos
+        id: get-changed-demos
+        uses: ./.github/actions/get-changed-demos
+
+      - name: Output changed demos
+        run: |
+          echo "Updated Demos: ${{ steps.get-changed-demos.outputs.updated }}"
+          echo "Deleted Demos: ${{ steps.get-changed-demos.outputs.deleted }}"
+
+      - name: Exit if no changes
+        if: steps.get-changed-demos.outputs.updated == ''
+        run: |
+          echo "No changes found in demos. Exiting workflow."
+
+  # Step 2: Build demos
+  build:
+    if: needs.identify-changed-demos.outputs.updated != ''
+    uses: ./.github/workflows/v2-build-demos.yml
+    needs: 
+      - identify-changed-demos
+    with:
+      ref: ${{ github.event.pull_request.head.sha }}
+      demo-names: ${{ needs.identify-changed-demos.outputs.updated }}
+      dev: ${{ github.event.pull_request.base.ref == 'dev' }}
+      save-artifact: true
+      artifact-name: demo-build-${{ github.event.pull_request.head.sha }}
+      keep-going: false
+      quiet: false
+      batch_size: 10
+
+  # Step 3: Save build context
+  save-build-context:
+    runs-on: ubuntu-latest
+    needs:
+      - build
+      - identify-changed-demos
+    steps:
+      - name: Save Pull Request Event Context
+        run: |
+          mkdir -p /tmp/pr
+          cat >/tmp/pr/pr_info.json <<EOL
+          {
+            "id": "${{ github.event.pull_request.number }}",
+            "ref": "${{ github.event.pull_request.head.sha }}",
+            "ref_name": "${{ github.event.pull_request.head.ref }}",
+            "updated_demos": "${{ needs.identify-changed-demos.outputs.updated }}",
+            "deleted_demos": "${{ needs.identify-changed-demos.outputs.deleted }}"
+          }
+          EOL
+
+      - name: Upload Pull Request Event Context as Artifact
+        uses: actions/upload-artifact@v4
+        with:
+          name: pr_info
+          path: /tmp/pr
+          retention-days: 30

.github/workflows/v2-deploy-demos.yml

@@ -21,7 +21,7 @@ on:
         type: environment
         required: true
       artifact-name:
-        description: Name of artifact containing demo builds
+        description: Name of artifact containing demo zip files.
         type: string
       preview:
         description: Whether to deploy demos as preview
@@ -38,7 +38,6 @@ jobs:
       - name: Checkout
         uses: actions/checkout@v4
         with:
-          ref: ${{ inputs.branch }}
           fetch-depth: 1
 
       - name: Configure AWS credentials
@@ -67,4 +66,3 @@ jobs:
           python3 .github/workflows/qml_pipeline_v2/deploy.py \
             --preview ${{ inputs.preview }} \
             _build/pack/*.zip
-