Skip to content

Commit 694bc64

Browse files
Json-To-StringJson-To-String
committed
Update workflows to dedicate runners per package per no/arch
1 parent 4514bd9 commit 694bc64

3 files changed

Lines changed: 135 additions & 69 deletions

File tree

‎.github/workflows/gen_matrix.py‎

Lines changed: 37 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,37 @@
1+
import toml
2+
import json
3+
4+
packages = toml.load('pkgs.toml')
5+
defaults = packages['defaults']
6+
7+
PLATFORM_OS_MAP = {
8+
'win-64': {'os': 'windows-latest', 'arch': 'x64'},
9+
'osx-arm64': {'os': 'macos-latest', 'arch': 'arm64'},
10+
'linux': {'os': 'ubuntu-latest', 'arch': 'x64'},
11+
}
12+
13+
arch_include = []
14+
noarch_include = []
15+
16+
for name, spec in packages.items():
17+
if name == 'defaults':
18+
continue
19+
noarch = spec.get('noarch', defaults['noarch'])
20+
if noarch:
21+
noarch_include.append({'name': name})
22+
else:
23+
platforms = spec.get('platforms', defaults['platforms'])
24+
pythons = spec.get('pythons', defaults['pythons'])
25+
for platform in platforms:
26+
os_info = PLATFORM_OS_MAP[platform]
27+
for python in pythons:
28+
arch_include.append({
29+
'name': name,
30+
'os': os_info['os'],
31+
'arch': os_info['arch'],
32+
'python': python,
33+
'platform': platform,
34+
})
35+
36+
print(f"arch_matrix={json.dumps({'include': arch_include})}")
37+
print(f"noarch_matrix={json.dumps({'include': noarch_include})}")

‎.github/workflows/make_packages.yml‎

Lines changed: 87 additions & 67 deletions
Original file line numberDiff line numberDiff line change
@@ -2,62 +2,53 @@ name: Make and upload Conda packages
22

33
on:
44
push:
5-
branches:
6-
- main
7-
tags:
8-
- '*'
5+
branches: [update-workflows]
6+
tags: ['*']
97
schedule:
10-
# - cron: '0-59/5 * * * *' # Every 5 minutes
11-
- cron: '13 02 * * 1' # Weekly, 2:13AM Mondays UTC
8+
- cron: '13 02 * * 1' # Weekly, 2:13 AM Mondays UTC
129

1310
env:
1411
ANACONDA_USER: rydiqule
1512

1613
jobs:
17-
build:
18-
name: Build
14+
15+
# ── Step 1: read pkgs.toml and emit two matrices ───────────────────────────
16+
configure:
17+
name: Configure build matrices
18+
runs-on: ubuntu-latest
19+
outputs:
20+
arch_matrix: ${{ steps.gen.outputs.arch_matrix }}
21+
noarch_matrix: ${{ steps.gen.outputs.noarch_matrix }}
22+
steps:
23+
- uses: actions/checkout@v4
24+
- uses: actions/setup-python@v5
25+
with:
26+
python-version: '3.x'
27+
- run: pip install toml
28+
- id: gen
29+
run: python .github/workflows/gen_matrix.py >> "$GITHUB_OUTPUT"
30+
31+
# ── Step 2a: one runner per (package × os × python) ────────────────────────
32+
build-arch:
33+
name: Build ${{ matrix.name }} – ${{ matrix.os }} py${{ matrix.python }}
34+
needs: configure
1935
runs-on: ${{ matrix.os }}
2036
strategy:
2137
fail-fast: false
22-
matrix:
23-
include:
24-
- { os: ubuntu-latest, python: '3.14', arch: x64}
25-
- { os: ubuntu-latest, python: '3.13', arch: x64}
26-
- { os: ubuntu-latest, python: '3.12', arch: x64}
27-
- { os: ubuntu-latest, python: '3.11', arch: x64}
28-
- { os: ubuntu-latest, python: '3.10', arch: x64 }
29-
30-
- { os: macos-latest, python: '3.14', arch: arm64}
31-
- { os: macos-latest, python: '3.13', arch: arm64}
32-
- { os: macos-latest, python: '3.12', arch: arm64}
33-
- { os: macos-latest, python: '3.11', arch: arm64}
34-
- { os: macos-latest, python: '3.10', arch: arm64}
35-
36-
- { os: windows-latest, python: '3.14', arch: x64 }
37-
- { os: windows-latest, python: '3.13', arch: x64 }
38-
- { os: windows-latest, python: '3.12', arch: x64 }
39-
- { os: windows-latest, python: '3.11', arch: x64 }
40-
- { os: windows-latest, python: '3.10', arch: x64 }
38+
matrix: ${{ fromJSON(needs.configure.outputs.arch_matrix) }}
4139

4240
if: github.repository == 'QTC-UMD/rydiqule-vendored-conda-builds'
4341
steps:
44-
- name: Checkout
45-
uses: actions/checkout@v6
42+
- uses: actions/checkout@v4
4643
with:
4744
fetch-depth: 0
4845

49-
- name: Checkout latest tag
46+
- name: Checkout latest tag (scheduled runs)
5047
if: github.event_name == 'schedule'
5148
shell: bash -l {0}
5249
run: git checkout $(git describe --tags `git rev-list --tags --max-count=1`)
5350

54-
- name: Set BUILD_NOARCH=true
55-
if: strategy.job-index == 0
56-
shell: bash -l {0}
57-
run: echo "BUILD_NOARCH=true" >> $GITHUB_ENV
58-
59-
- name: Install Miniforge
60-
uses: conda-incubator/setup-miniconda@v4
51+
- uses: conda-incubator/setup-miniconda@v3
6152
with:
6253
miniforge-version: "latest"
6354
channels: conda-forge
@@ -67,42 +58,83 @@ jobs:
6758
architecture: ${{ matrix.arch }}
6859
auto-activate: true
6960

70-
- name: Conda package (Unix)
61+
- name: Build (Unix)
7162
if: runner.os != 'Windows'
7263
shell: bash -l {0}
7364
run: |
7465
conda install -c labscript-suite setuptools-conda
75-
python make_packages.py ${{ runner.temp }}
66+
python make_packages.py ${{ runner.temp }} --package "${{ matrix.name }}"
7667
77-
- name: Conda Package (Windows)
68+
- name: Build (Windows)
7869
if: runner.os == 'Windows'
7970
shell: cmd /C CALL {0}
8071
run: |
8172
conda install -c labscript-suite setuptools-conda && ^
82-
python make_packages.py ${{ runner.temp }}
73+
python make_packages.py ${{ runner.temp }} --package "${{ matrix.name }}"
74+
75+
- uses: actions/upload-artifact@v4
76+
with:
77+
name: conda_packages-${{ matrix.name }}-${{ matrix.os }}-py${{ matrix.python }}-${{ matrix.arch }}
78+
path: ./conda_packages
79+
80+
# ── Step 2b: one runner per noarch package (OS/Python don't matter) ─────────
81+
build-noarch:
82+
name: Build ${{ matrix.name }} – noarch
83+
needs: configure
84+
runs-on: ubuntu-latest
85+
strategy:
86+
fail-fast: false
87+
matrix: ${{ fromJSON(needs.configure.outputs.noarch_matrix) }}
88+
89+
if: github.repository == 'QTC-UMD/rydiqule-vendored-conda-builds'
90+
steps:
91+
- uses: actions/checkout@v4
92+
with:
93+
fetch-depth: 0
94+
95+
- name: Checkout latest tag (scheduled runs)
96+
if: github.event_name == 'schedule'
97+
shell: bash -l {0}
98+
run: git checkout $(git describe --tags `git rev-list --tags --max-count=1`)
8399

84-
- name: Upload Artifact
85-
uses: actions/upload-artifact@v7
86-
with: # v4 requires all artifact names are unique on upload
87-
name: conda_packages-${{ matrix.os }}-py${{ matrix.python }}-${{ matrix.arch }}
100+
- uses: conda-incubator/setup-miniconda@v3
101+
with:
102+
miniforge-version: "latest"
103+
channels: conda-forge
104+
auto-update-conda: true
105+
conda-remove-defaults: true
106+
auto-activate: true
107+
108+
- name: Build noarch package
109+
shell: bash -l {0}
110+
run: |
111+
conda install -c labscript-suite setuptools-conda
112+
python make_packages.py ${{ runner.temp }} --package "${{ matrix.name }}"
113+
114+
- uses: actions/upload-artifact@v4
115+
with:
116+
name: conda_packages-${{ matrix.name }}-noarch
88117
path: ./conda_packages
89118

119+
# ── Step 3: upload once all build jobs succeed (or at least one did) ────────
90120
upload:
91-
name: Upload
121+
name: Upload to Anaconda
92122
runs-on: ubuntu-latest
93-
needs: build
94-
if: ${{ always() && contains(join(needs.build.result, ','), 'success') }}
123+
needs: [build-arch, build-noarch]
124+
if: >-
125+
always() &&
126+
(
127+
contains(join(needs.build-arch.result, ','), 'success') ||
128+
contains(join(needs.build-noarch.result, ','), 'success')
129+
)
95130
steps:
96-
97-
- name: Download All Artifacts
98-
uses: actions/download-artifact@v8
131+
- uses: actions/download-artifact@v4
99132
with:
100133
pattern: conda_packages-*
101134
path: ./conda_packages
102135
merge-multiple: true
103136

104-
- name: Install Miniconda
105-
uses: conda-incubator/setup-miniconda@v4
137+
- uses: conda-incubator/setup-miniconda@v3
106138
with:
107139
miniforge-version: "latest"
108140
auto-update-conda: true
@@ -114,19 +146,7 @@ jobs:
114146
shell: bash -l {0}
115147
run: conda install anaconda-client
116148

117-
# - name: Publish to Anaconda test label
118-
# if: github.event.ref_type != 'tag' && github.event_name != 'schedule'
119-
# shell: bash -l {0}
120-
# run: |
121-
# anaconda \
122-
# --token ${{ secrets.ANACONDA_API_TOKEN }} \
123-
# upload \
124-
# --user $ANACONDA_USER \
125-
# --label test \
126-
# --skip-existing \
127-
# conda_packages/*/*
128-
129-
- name: Publish to Anaconda main label
149+
- name: Publish to Anaconda (main label)
130150
if: (github.event_name == 'push' && contains(github.ref, '/tags')) || github.event_name == 'schedule'
131151
shell: bash -l {0}
132152
run: |
@@ -135,4 +155,4 @@ jobs:
135155
upload \
136156
--user $ANACONDA_USER \
137157
--skip-existing \
138-
conda_packages/*/*
158+
conda_packages/*/*

‎make_packages.py‎

Lines changed: 11 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -147,12 +147,21 @@ def build_conda_package(name, spec):
147147

148148

149149
if __name__ == '__main__':
150+
import argparse
151+
parser = argparse.ArgumentParser()
152+
parser.add_argument('build_dir', nargs='?', default='build')
153+
parser.add_argument('--package', default=None,
154+
help='Only build this specific package (by TOML key)')
155+
args = parser.parse_args()
156+
BUILD_DIR = args.build_dir # reassign the module-level var
157+
150158
Path(BUILD_DIR).mkdir(exist_ok=True)
151159
packages = toml.load('pkgs.toml')
152160
defaults = packages['defaults']
153161

154162
for name, spec in packages.items():
155163
if name == 'defaults':
156164
continue
157-
158-
build_conda_package(name, spec)
165+
if args.package and name != args.package:
166+
continue
167+
build_conda_package(name, spec)

0 commit comments

Comments
 (0)