data-science-pipelines/.github/workflows/kfp-samples.yml at 540843bbef1f7326720b78627e6f2c0e3584f97b · VaniHaripriya/data-science-pipelines · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
name: KFP Samples

on:
  push:
    branches:
      - master
      - main
      - stable
      - 'rhoai-*'
  pull_request:
    paths:
      - '.github/resources/**'
      - '.github/workflows/kfp-samples.yml'
      - '.github/workflows/kubeflow-pipelines-integration-v2.yml'
      - 'backend/**'
      - 'samples/**'
      - 'samples/core/dataflow/**'
      - 'samples/core/parameterized_tfx_oss/**'
      - '!**/*.md'
      - '!**/OWNERS'

jobs:
  samples:
    runs-on: ubuntu-latest
    strategy:
      matrix:
        k8s_version: [ "v1.29.2", "v1.30.2", "v1.31.0" ]
    name: KFP Samples - K8s ${{ matrix.k8s_version }}

    steps:
    - name: Checkout code
      uses: actions/checkout@v2

    - name: Set up Python
      uses: actions/setup-python@v4
      with:
        python-version: 3.9

    - name: apt-get update
      run: sudo apt-get update

    - name: Install protobuf-compiler
      run: sudo apt-get install protobuf-compiler -y

    - name: Install setuptools
      run: |
        pip3 install setuptools
        pip3 freeze

    - name: Install Wheel
      run: pip3 install wheel==0.42.0

    - name: Install protobuf
      run: pip3 install protobuf==4.25.3

    - name: Generate API proto files
      working-directory: ./api
      run: make python

    - name: Install kfp-pipeline-spec from source
      run: |
        python3 -m pip install api/v2alpha1/python

    - name: Generate, Build, and Install Kubernetes API proto files & packages
      working-directory: ./kubernetes_platform
      run: make python && pip install python/dist/*.whl

    - name: Create KFP cluster
      id: create-kfp-cluster
      uses: ./.github/actions/kfp-cluster
      with:
        k8s_version: ${{ matrix.k8s_version }}
      continue-on-error: true

    - name: Build and upload the sample Modelcar image to Kind
      id: build-sample-modelcar-image
      if: ${{ (steps.create-kfp-cluster.outcome == 'success' )}}
      run: |
        docker build -f samples/v2/modelcar/Dockerfile -t registry.domain.local/modelcar:test .
        kind --name kfp load docker-image registry.domain.local/modelcar:test
      continue-on-error: true

    - name: Forward API port
      id: forward-api-port
      if: ${{ (steps.build-sample-modelcar-image.outcome == 'success' )}}
      run: ./.github/resources/scripts/forward-port.sh "kubeflow" "ml-pipeline" 8888 8888

    - name: Run Samples Tests
      id: tests
      if: ${{ (steps.forward-api-port.outcome == 'success' )}}
      env:
        PULL_NUMBER: ${{ github.event.pull_request.number }}
        REPO_NAME: ${{ github.repository }}
      run: |
        ./backend/src/v2/test/sample-test.sh
      continue-on-error: true

    - name: Collect failed logs
      if: ${{ (steps.create-kfp-cluster.outcome != 'success' ) || ( steps.tests.outcome != 'success' )}}
      run: |
        ./.github/resources/scripts/collect-logs.sh --ns kubeflow --output /tmp/tmp_pod_log.txt
        exit 1

    - name: Collect test results
      if: always()
      uses: actions/upload-artifact@v4
      with:
        name: kfp-samples-tests-artifacts-k8s-${{ matrix.k8s_version }}
        path: /tmp/tmp*/*