Skip to content

Commit d0bee95

Browse files
avik-palavik-pal
committed
ci: improved testing by auto-parsing sources
1 parent 97b16dd commit d0bee95

File tree

9 files changed

+89
-55
lines changed

9 files changed

+89
-55
lines changed

‎.github/workflows/CI.yml‎

Lines changed: 1 addition & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -47,8 +47,6 @@ jobs:
4747
project: "."
4848
test_args: "BACKEND_GROUP=CPU LUX_TEST_GROUP=${{ matrix.test_group }}"
4949
os: ${{ matrix.os }}
50-
local_dependencies: "lib/MLDataDevices,lib/WeightInitializers,lib/LuxLib,lib/LuxCore"
51-
local_test_dependencies: "lib/MLDataDevices,lib/LuxTestUtils,lib/LuxLib,lib/LuxCore"
5250

5351
downgrade:
5452
strategy:
@@ -64,9 +62,7 @@ jobs:
6462
- "extras"
6563
uses: ./.github/workflows/CommonCI.yml
6664
with:
67-
julia_version: "1.11"
65+
julia_version: "1.10"
6866
project: "."
6967
downgrade_testing: true
7068
test_args: "BACKEND_GROUP=CPU LUX_TEST_GROUP=${{ matrix.test_group }}"
71-
local_dependencies: "lib/MLDataDevices,lib/WeightInitializers,lib/LuxLib,lib/LuxCore"
72-
local_test_dependencies: "lib/MLDataDevices,lib/LuxTestUtils,lib/LuxLib,lib/LuxCore"

‎.github/workflows/CI_LuxCUDA.yml‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -16,15 +16,15 @@ concurrency:
1616
cancel-in-progress: ${{ startsWith(github.ref, 'refs/pull/') }}
1717

1818
jobs:
19-
downgrade:
19+
test:
2020
uses: ./.github/workflows/CommonCI.yml
2121
with:
22-
julia_version: "1.11"
22+
julia_version: "1.12"
2323
project: "lib/LuxCUDA"
24-
downgrade_testing: true
2524

26-
test:
25+
downgrade:
2726
uses: ./.github/workflows/CommonCI.yml
2827
with:
29-
julia_version: "1.11"
28+
julia_version: "1.10"
3029
project: "lib/LuxCUDA"
30+
downgrade_testing: true

‎.github/workflows/CI_LuxCore.yml‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -26,7 +26,6 @@ jobs:
2626
downgrade:
2727
uses: ./.github/workflows/CommonCI.yml
2828
with:
29-
julia_version: "1.11"
29+
julia_version: "1.10"
3030
project: "lib/LuxCore"
3131
downgrade_testing: true
32-
local_test_dependencies: "lib/MLDataDevices"

‎.github/workflows/CI_LuxLib.yml‎

Lines changed: 1 addition & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -54,9 +54,7 @@ jobs:
5454
- "misc"
5555
uses: ./.github/workflows/CommonCI.yml
5656
with:
57-
julia_version: "1.11"
57+
julia_version: "1.10"
5858
project: "lib/LuxLib"
5959
downgrade_testing: true
60-
local_dependencies: "lib/LuxCore,lib/MLDataDevices"
61-
local_test_dependencies: "lib/LuxTestUtils,lib/MLDataDevices"
6260
test_args: "BACKEND_GROUP=cpu LUXLIB_TEST_GROUP=${{ matrix.test_group }} LUXLIB_BLAS_BACKEND=default LUXLIB_LOAD_LOOPVEC=true"

‎.github/workflows/CI_MLDataDevices.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -45,7 +45,7 @@ jobs:
4545
- reactant
4646
uses: ./.github/workflows/CommonCI.yml
4747
with:
48-
julia_version: "1.11"
48+
julia_version: "1.10"
4949
project: "lib/MLDataDevices"
5050
downgrade_testing: true
5151
test_args: "BACKEND_GROUP=${{ matrix.group }}"

‎.github/workflows/CI_WeightInitializers.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -26,7 +26,7 @@ jobs:
2626
downgrade:
2727
uses: ./.github/workflows/CommonCI.yml
2828
with:
29-
julia_version: "1.11"
29+
julia_version: "1.10"
3030
project: "lib/WeightInitializers"
3131
downgrade_testing: true
3232
test_args: "BACKEND_GROUP=cpu"

‎.github/workflows/CommonCI.yml‎

Lines changed: 72 additions & 31 deletions
Original file line numberDiff line numberDiff line change
@@ -21,16 +21,6 @@ on:
2121
required: false
2222
type: boolean
2323
default: false
24-
local_dependencies:
25-
description: "Local Dependencies"
26-
required: false
27-
type: string
28-
default: ""
29-
local_test_dependencies:
30-
description: "Local Test Dependencies"
31-
required: false
32-
type: string
33-
default: ""
3424
test_args:
3525
description: "Test Arguments"
3626
required: false
@@ -61,55 +51,106 @@ jobs:
6151
version: ${{ inputs.julia_version }}
6252
- uses: julia-actions/cache@v2
6353
with:
64-
cache-name: julia-cache;workflow=${{ inputs.julia_version }}-${{ inputs.os }}-${{ github.event_name }}-${{ inputs.project }}-${{ inputs.downgrade_testing }}-${{ inputs.local_dependencies }}-${{ inputs.local_test_dependencies }}-${{ inputs.test_args }}
54+
cache-name: julia-cache;workflow=${{ inputs.julia_version }}-${{ inputs.os }}-${{ github.event_name }}-${{ inputs.project }}-${{ inputs.downgrade_testing }}-${{ inputs.test_args }}
6555

6656
- uses: julia-actions/julia-downgrade-compat@v2.1
6757
if: ${{ inputs.downgrade_testing }}
6858
with:
6959
mode: forcedeps
70-
projects: ${{ inputs.project }}
60+
projects: ${{ inputs.project }}, ${{ inputs.project }}/test
7161
julia_version: ${{ inputs.julia_version }}
7262
skip: Pkg, TOML, Statistics, LinearAlgebra, Random, Serialization, Markdown, Test, LuxCore, LuxLib, LuxTestUtils, MLDataDevices, WeightInitializers, UUIDs
7363

74-
# For 1.10 we need to manually develop the packages.
64+
# For 1.10 we need to manually develop the packages from [sources].
7565
- name: "Develop Dependencies"
7666
run: |
7767
import Pkg
78-
Pkg.activate(joinpath(pwd(), get(ENV, "PROJECT", "")))
68+
import TOML
69+
70+
project = joinpath(pwd(), get(ENV, "PROJECT", ""))
71+
Pkg.activate(project)
72+
Pkg.status()
73+
Pkg.instantiate()
74+
75+
# Parse the [sources] section from the Project.toml
76+
project_toml = TOML.parsefile(joinpath(project, "Project.toml"))
77+
sources = get(project_toml, "sources", Dict())
78+
7979
dev_pkgs = Pkg.PackageSpec[]
80-
for pkg in filter(!isempty, split(get(ENV, "DEV_DEPS", ""), ","))
81-
push!(dev_pkgs, Pkg.PackageSpec(path=pkg))
80+
for (pkg, source) in sources
81+
if haskey(source, "path")
82+
pkg_path = joinpath(project, source["path"])
83+
push!(dev_pkgs, Pkg.PackageSpec(path=pkg_path))
84+
end
85+
end
86+
87+
if !isempty(dev_pkgs)
88+
@info "Developing dependencies: $dev_pkgs"
89+
if parse(Bool, get(ENV, "DOWNGRADE_TESTING", "false"))
90+
@info "Preserving dependencies"
91+
Pkg.develop(dev_pkgs; preserve=Pkg.PRESERVE_ALL)
92+
else
93+
Pkg.develop(dev_pkgs)
94+
end
8295
end
83-
length(dev_pkgs) > 0 && Pkg.develop(dev_pkgs)
8496
shell: julia --color=yes --threads=auto {0}
85-
if: ${{ (inputs.julia_version == '1.10' || inputs.julia_version == 'lts') && inputs.local_dependencies != '' }}
97+
if: ${{ inputs.julia_version == '1.10' || inputs.julia_version == 'lts' }}
8698
env:
8799
PROJECT: ${{ inputs.project }}
88-
DEV_DEPS: ${{ inputs.local_dependencies }}
100+
DOWNGRADE_TESTING: ${{ inputs.downgrade_testing }}
101+
JULIA_PKG_PRECOMPILE_AUTO: 0
89102

90-
# For 1.11 and beyond we can use sources. For older versions we need to
91-
# manually develop the packages.
103+
# For 1.11 and beyond we can use sources.
92104
- uses: julia-actions/julia-buildpkg@v1
93105
with:
94106
project: ${{ inputs.project }}
95107
precompile: true
96-
if: ${{ (inputs.julia_version != '1.10' && inputs.julia_version != 'lts') || inputs.local_dependencies == '' }}
108+
if: ${{ inputs.julia_version != '1.10' && inputs.julia_version != 'lts' }}
97109

98110
- name: "Develop Test Dependencies"
99111
run: |
100112
import Pkg
113+
import TOML
114+
101115
project = joinpath(pwd(), get(ENV, "PROJECT", ""))
102-
Pkg.activate(joinpath(project, "test"))
103-
dev_pkgs = Pkg.PackageSpec[Pkg.PackageSpec(path=project)]
104-
for pkg in filter(!isempty, split(get(ENV, "TEST_DEPS", ""), ","))
105-
push!(dev_pkgs, Pkg.PackageSpec(path=pkg))
116+
test_project = joinpath(project, "test")
117+
Pkg.activate(test_project)
118+
Pkg.status()
119+
Pkg.instantiate()
120+
121+
# Add the main project itself
122+
dev_pkgs = Pkg.PackageSpec[]
123+
124+
# Parse the [sources] section from the test/Project.toml
125+
test_project_toml_path = joinpath(test_project, "Project.toml")
126+
if isfile(test_project_toml_path)
127+
test_project_toml = TOML.parsefile(test_project_toml_path)
128+
sources = get(test_project_toml, "sources", Dict())
129+
130+
for (pkg, source) in sources
131+
if haskey(source, "path")
132+
pkg_path = joinpath(test_project, source["path"])
133+
push!(dev_pkgs, Pkg.PackageSpec(path=pkg_path))
134+
end
135+
end
136+
end
137+
138+
if !isempty(dev_pkgs)
139+
@info "Developing dependencies: $dev_pkgs"
140+
if parse(Bool, get(ENV, "DOWNGRADE_TESTING", "false"))
141+
@info "Preserving dependencies"
142+
Pkg.develop(dev_pkgs; preserve=Pkg.PRESERVE_ALL)
143+
else
144+
Pkg.develop(dev_pkgs)
145+
end
106146
end
107-
length(dev_pkgs) > 0 && Pkg.develop(dev_pkgs)
108147
shell: julia --color=yes --threads=auto {0}
109-
if: ${{ (inputs.julia_version == '1.10' || inputs.julia_version == 'lts') && inputs.local_test_dependencies != '' }}
148+
if: ${{ inputs.julia_version == '1.10' || inputs.julia_version == 'lts' }}
110149
env:
111150
PROJECT: ${{ inputs.project }}
112-
TEST_DEPS: ${{ inputs.local_test_dependencies }}
151+
DOWNGRADE_TESTING: ${{ inputs.downgrade_testing }}
152+
JULIA_PKG_PRECOMPILE_AUTO: 0
153+
113154
- name: "Run Tests"
114155
run: |
115156
import Pkg
@@ -118,7 +159,7 @@ jobs:
118159
const TEST_ARGS = filter(!isempty, split(get(ENV, "TEST_ARGS", "")))
119160
include(joinpath(project, "test", "runtests.jl"))
120161
shell: julia --color=yes --code-coverage=user --depwarn=yes --threads=auto --check-bounds=yes {0}
121-
if: ${{ (inputs.julia_version == '1.10' || inputs.julia_version == 'lts') && inputs.local_test_dependencies != '' }}
162+
if: ${{ inputs.julia_version == '1.10' || inputs.julia_version == 'lts' }}
122163
env:
123164
PROJECT: ${{ inputs.project }}
124165
TEST_ARGS: ${{ inputs.test_args }}
@@ -128,7 +169,7 @@ jobs:
128169
project: ${{ inputs.project }}
129170
test_args: ${{ inputs.test_args }}
130171
allow_reresolve: ${{ !(inputs.downgrade_testing) }}
131-
if: ${{ (inputs.julia_version != '1.10' && inputs.julia_version != 'lts') || inputs.local_test_dependencies == '' }}
172+
if: ${{ inputs.julia_version != '1.10' && inputs.julia_version != 'lts' }}
132173

133174
- name: "Upload MLIR modules"
134175
uses: actions/upload-artifact@v6

‎lib/LuxLib/test/Project.toml‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -52,7 +52,7 @@ EnzymeCore = "0.8.14"
5252
ExplicitImports = "1.9.0"
5353
ForwardDiff = "0.10.36, =1"
5454
InteractiveUtils = "<0.0.1, 1"
55-
JLArrays = "0.1.5, 0.2, 0.3"
55+
JLArrays = "0.2, 0.3"
5656
LoopVectorization = "0.12.171"
5757
LuxTestUtils = "2"
5858
MKL = "0.7, 0.8, 0.9"
@@ -71,5 +71,5 @@ Static = "0.8.4, 1"
7171
StaticArrays = "1.9.7"
7272
Statistics = "1.10"
7373
Test = "1.10"
74-
Tracker = "0.2.36"
74+
Tracker = "0.2.37"
7575
Zygote = "0.7"

‎lib/MLDataDevices/test/Project.toml‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -27,23 +27,23 @@ LuxTestUtils = {path = "../../LuxTestUtils"}
2727
MLDataDevices = {path = ".."}
2828

2929
[compat]
30-
Adapt = "4"
30+
Adapt = "4.1"
3131
Aqua = "0.8.4"
3232
ArrayInterface = "7.17.1"
3333
ChainRulesTestUtils = "1.13.0"
3434
ComponentArrays = "0.15.22"
3535
ExplicitImports = "1.9.0"
36-
FillArrays = "1"
36+
FillArrays = "1.13"
3737
ForwardDiff = "0.10.36, 1"
3838
Functors = "0.5"
3939
MLUtils = "0.4.4"
40-
OneHotArrays = "0.2.5"
40+
OneHotArrays = "0.2.10"
4141
Pkg = "1.10"
4242
Random = "1.10"
4343
Reactant = "0.2.170"
44-
RecursiveArrayTools = "3.8"
44+
RecursiveArrayTools = "3.28"
4545
ReverseDiff = "1.15"
4646
SparseArrays = "1.10"
4747
Test = "1.10"
48-
Tracker = "0.2.36"
48+
Tracker = "0.2.37"
4949
Zygote = "0.7"

0 commit comments

Comments
 (0)