Skip to content

Commit 32ed928

Browse files
lithoruslithorus
authored
Add build pipeline for packaging all OpenCue modules in CI workflow (#1785)
This is to fix the release pipeline. It takes the same parts of the testing pipeline to first build the python packages and use them for building the docker images. I have tested the integration tests that seems to work, but I cannot test the other steps, which requires AWS credentials. This is needed for #1779 to work properly and generally to allow the packaging to work again after the rework of the build setup for the python libraries.
1 parent 3d88ef1 commit 32ed928

File tree

2 files changed

+111
-10
lines changed

2 files changed

+111
-10
lines changed

‎.github/workflows/packaging-pipeline.yml‎

Lines changed: 97 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -6,15 +6,107 @@ on:
66
branches: [ master ]
77

88
jobs:
9+
build_opencue_packages:
10+
name: Build opencue packages
11+
runs-on: ubuntu-22.04
12+
container: python:3.7
13+
outputs:
14+
opencue_proto_path: ${{ steps.build_opencue_proto.outputs.opencue_proto_path }}
15+
opencue_pycue_path: ${{ steps.build_pycue.outputs.opencue_pycue_path }}
16+
opencue_pyoutline_path: ${{ steps.build_pyoutline.outputs.opencue_pyoutline_path }}
17+
opencue_cueadmin_path: ${{ steps.build_cueadmin.outputs.opencue_cueadmin_path }}
18+
opencue_cuesubmit_path: ${{ steps.build_cuesubmit.outputs.opencue_cuesubmit_path }}
19+
opencue_rqd_path: ${{ steps.build_rqd.outputs.opencue_rqd_path }}
20+
steps:
21+
- uses: actions/checkout@v4
22+
with:
23+
fetch-tags: true
24+
fetch-depth: 0
25+
- name: Mark repository as safe (Fix for https://github.com/actions/checkout/issues/1048)
26+
run: git config --global --add safe.directory $GITHUB_WORKSPACE
27+
28+
- name: Prepare building packages
29+
run: |
30+
pip install build==1.1.1
31+
32+
- name: Build opencue_proto package
33+
id: build_opencue_proto
34+
run: |
35+
set -e
36+
python -m build ./proto
37+
echo "opencue_proto_path=$(find ./proto/dist -name 'opencue_proto-*.whl' -print -quit)" >> $GITHUB_OUTPUT
38+
39+
- name: Build opencue_pycue package
40+
id: build_pycue
41+
run: |
42+
set -e
43+
python -m build ./pycue
44+
echo "opencue_pycue_path=$(find ./pycue/dist -name 'opencue_pycue-*.whl' -print -quit)" >> $GITHUB_OUTPUT
45+
46+
- name: Build opencue_pyoutline package
47+
id: build_pyoutline
48+
run: |
49+
set -e
50+
python -m build ./pyoutline
51+
echo "opencue_pyoutline_path=$(find ./pyoutline/dist -name 'opencue_pyoutline-*.whl' -print -quit)" >> $GITHUB_OUTPUT
52+
53+
- name: Build opencue_cueadmin package
54+
id: build_cueadmin
55+
run: |
56+
set -e
57+
python -m build ./cueadmin
58+
echo "opencue_cueadmin_path=$(find ./cueadmin/dist -name 'opencue_cueadmin-*.whl' -print -quit)" >> $GITHUB_OUTPUT
59+
60+
- name: Build opencue_cuesubmit package
61+
id: build_cuesubmit
62+
run: |
63+
set -e
64+
python -m build ./cuesubmit
65+
echo "opencue_cuesubmit_path=$(find ./cuesubmit/dist -name 'opencue_cuesubmit-*.whl' -print -quit)" >> $GITHUB_OUTPUT
66+
67+
- name: Build opencue_rqd package
68+
id: build_rqd
69+
run: |
70+
set -e
71+
python -m build ./rqd
72+
echo "opencue_rqd_path=$(find ./rqd/dist -name 'opencue_rqd-*.whl' -print -quit)" >> $GITHUB_OUTPUT
73+
74+
- name: Upload opencue packages
75+
uses: actions/upload-artifact@v4
76+
with:
77+
name: opencue_packages
78+
path: |
79+
proto/dist/*.*
80+
pycue/dist/*.*
81+
pyoutline/dist/*.*
82+
cueadmin/dist/*.*
83+
cuesubmit/dist/*.*
84+
rqd/dist/*.*
985
integration_test:
86+
needs: build_opencue_packages
1087
name: Run Integration Test
1188
runs-on: ubuntu-22.04
1289
steps:
1390
- name: Checkout
14-
uses: actions/checkout@v3
15-
91+
uses: actions/checkout@v4
92+
with:
93+
fetch-depth: 0
94+
fetch-tags: true
95+
- name: Mark repository as safe (Fix for https://github.com/actions/checkout/issues/1048)
96+
run: git config --global --add safe.directory $GITHUB_WORKSPACE
97+
- name: Download a single artifact
98+
uses: actions/download-artifact@v4
99+
with:
100+
name: opencue_packages
16101
- name: Run test
17-
run: ci/run_integration_test.sh
102+
run: |
103+
export OPENCUE_PROTO_PACKAGE_PATH="${{ needs.build_opencue_packages.outputs.opencue_proto_path }}"
104+
export OPENCUE_PYCUE_PACKAGE_PATH="${{ needs.build_opencue_packages.outputs.opencue_pycue_path }}"
105+
export OPENCUE_PYOUTLINE_PACKAGE_PATH="${{ needs.build_opencue_packages.outputs.opencue_pyoutline_path }}"
106+
export OPENCUE_CUEADMIN_PACKAGE_PATH="${{ needs.build_opencue_packages.outputs.opencue_cueadmin_path }}"
107+
export OPENCUE_CUESUBMIT_PACKAGE_PATH="${{ needs.build_opencue_packages.outputs.opencue_cuesubmit_path }}"
108+
export OPENCUE_RQD_PACKAGE_PATH="${{ needs.build_opencue_packages.outputs.opencue_rqd_path }}"
109+
ci/run_integration_test.sh
18110
19111
- name: Archive log files
20112
uses: actions/upload-artifact@v4
@@ -59,6 +151,7 @@ jobs:
59151

60152
name: Build ${{ matrix.NAME }}
61153
runs-on: ubuntu-22.04
154+
continue-on-error: true
62155
steps:
63156
- name: Checkout
64157
uses: actions/checkout@v3
@@ -122,6 +215,7 @@ jobs:
122215
name: Create Other Build Artifacts
123216
needs: build_components
124217
runs-on: ubuntu-22.04
218+
continue-on-error: true
125219
steps:
126220
- name: Checkout
127221
uses: actions/checkout@v3

‎.github/workflows/testing-pipeline.yml‎

Lines changed: 14 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@ jobs:
1818
opencue_cueadmin_path: ${{ steps.build_cueadmin.outputs.opencue_cueadmin_path }}
1919
opencue_cuesubmit_path: ${{ steps.build_cuesubmit.outputs.opencue_cuesubmit_path }}
2020
opencue_rqd_path: ${{ steps.build_rqd.outputs.opencue_rqd_path }}
21+
2122
steps:
2223
- uses: actions/checkout@v4
2324
with:
@@ -28,43 +29,49 @@ jobs:
2829

2930
- name: Prepare building packages
3031
run: |
31-
pip install build
32+
pip install build==1.1.1
3233
3334
- name: Build opencue_proto package
3435
id: build_opencue_proto
3536
run: |
37+
set -e
3638
python -m build ./proto
37-
echo "opencue_proto_path=$(ls ./proto/dist/opencue_proto-*.whl)" >> $GITHUB_OUTPUT
39+
echo "opencue_proto_path=$(find ./proto/dist -name 'opencue_proto-*.whl' -print -quit)" >> $GITHUB_OUTPUT
3840
3941
- name: Build opencue_pycue package
4042
id: build_pycue
4143
run: |
44+
set -e
4245
python -m build ./pycue
43-
echo "opencue_pycue_path=$(ls ./pycue/dist/opencue_pycue-*.whl)" >> $GITHUB_OUTPUT
46+
echo "opencue_pycue_path=$(find ./pycue/dist -name 'opencue_pycue-*.whl' -print -quit)" >> $GITHUB_OUTPUT
4447
4548
- name: Build opencue_pyoutline package
4649
id: build_pyoutline
4750
run: |
51+
set -e
4852
python -m build ./pyoutline
49-
echo "opencue_pyoutline_path=$(ls ./pyoutline/dist/opencue_pyoutline-*.whl)" >> $GITHUB_OUTPUT
53+
echo "opencue_pyoutline_path=$(find ./pyoutline/dist -name 'opencue_pyoutline-*.whl' -print -quit)" >> $GITHUB_OUTPUT
5054
5155
- name: Build opencue_cueadmin package
5256
id: build_cueadmin
5357
run: |
58+
set -e
5459
python -m build ./cueadmin
55-
echo "opencue_cueadmin_path=$(ls ./cueadmin/dist/opencue_cueadmin-*.whl)" >> $GITHUB_OUTPUT
60+
echo "opencue_cueadmin_path=$(find ./cueadmin/dist -name 'opencue_cueadmin-*.whl' -print -quit)" >> $GITHUB_OUTPUT
5661
5762
- name: Build opencue_cuesubmit package
5863
id: build_cuesubmit
5964
run: |
65+
set -e
6066
python -m build ./cuesubmit
61-
echo "opencue_cuesubmit_path=$(ls ./cuesubmit/dist/opencue_cuesubmit-*.whl)" >> $GITHUB_OUTPUT
67+
echo "opencue_cuesubmit_path=$(find ./cuesubmit/dist -name 'opencue_cuesubmit-*.whl' -print -quit)" >> $GITHUB_OUTPUT
6268
6369
- name: Build opencue_rqd package
6470
id: build_rqd
6571
run: |
72+
set -e
6673
python -m build ./rqd
67-
echo "opencue_rqd_path=$(ls ./rqd/dist/opencue_rqd-*.whl)" >> $GITHUB_OUTPUT
74+
echo "opencue_rqd_path=$(find ./rqd/dist -name 'opencue_rqd-*.whl' -print -quit)" >> $GITHUB_OUTPUT
6875
6976
- name: Upload opencue packages
7077
uses: actions/upload-artifact@v4

0 commit comments

Comments
 (0)