Skip to content

Commit 4715e54

Browse files
gdallegiordanowsmosespenelopeysm
authored
Add DI integration tests (#2531)
* Add DI integration tests * Increase timeout * Remove JuliaFormatter * Add StaticArrays compat * Run Runic * Apply suggestions from code review Co-authored-by: Mosè Giordano <[email protected]> * Fix deps * Add container * Fix instantiation * Apply suggestion * Speed up tests by removing matrices * Format * Doc * Fix undefined `Scenarioù` * Reduce timeout, fix version * Mark broken Bijectors test (tentative) * Format * Same compiler flags * Update test/integration/Bijectors/runtests.jl Co-authored-by: Penelope Yong <[email protected]> * Update test/integration/Bijectors/runtests.jl Co-authored-by: Penelope Yong <[email protected]> * Update .github/workflows/Integration.yml Co-authored-by: Mosè Giordano <[email protected]> --------- Co-authored-by: Mosè Giordano <[email protected]> Co-authored-by: William Moses <[email protected]> Co-authored-by: Penelope Yong <[email protected]>
1 parent bc4d13b commit 4715e54

File tree

9 files changed

+230
-30
lines changed

9 files changed

+230
-30
lines changed

.github/workflows/CI.yml

Lines changed: 0 additions & 28 deletions
Original file line numberDiff line numberDiff line change
@@ -230,34 +230,6 @@ jobs:
230230
files: lcov.info
231231
token: ${{ secrets.CODECOV_TOKEN }}
232232
fail_ci_if_error: false # or true if you want CI to fail when Codecov fails
233-
integration:
234-
timeout-minutes: 20
235-
name: Integration Tests - ${{ matrix.test }}
236-
runs-on: ${{ matrix.os }}
237-
env:
238-
JULIA_PKG_SERVER_REGISTRY_PREFERENCE: eager
239-
strategy:
240-
fail-fast: false
241-
matrix:
242-
version:
243-
- '1.10'
244-
os:
245-
- ubuntu-latest
246-
test:
247-
- DynamicExpressions
248-
- Bijectors
249-
steps:
250-
- uses: actions/checkout@v4
251-
- uses: julia-actions/setup-julia@v2
252-
with:
253-
version: ${{ matrix.version }}
254-
- uses: julia-actions/cache@v2
255-
- uses: julia-actions/julia-buildpkg@v1
256-
- name: "Run tests"
257-
run: |
258-
julia --color=yes --project=test/integration/${{ matrix.test }} -e 'using Pkg; Pkg.develop([PackageSpec(; path) for path in (".", "lib/EnzymeCore")]); Pkg.instantiate()'
259-
julia --color=yes --project=test/integration/${{ matrix.test }} --threads=auto --check-bounds=yes test/integration/${{ matrix.test }}/runtests.jl
260-
shell: bash
261233
docs:
262234
timeout-minutes: 20
263235
name: Documentation

.github/workflows/Integration.yml

Lines changed: 70 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,70 @@
1+
name: Integration
2+
on:
3+
pull_request:
4+
paths:
5+
- '.github/workflows/Integration.yml'
6+
- 'ext/**'
7+
- 'lib/**'
8+
- 'src/**'
9+
- 'test/**'
10+
- 'Project.toml'
11+
push:
12+
branches:
13+
- main
14+
paths:
15+
- release-*
16+
- '.github/workflows/Integration.yml'
17+
- 'ext/**'
18+
- 'lib/**'
19+
- 'src/**'
20+
- 'test/**'
21+
- 'Project.toml'
22+
tags: '*'
23+
24+
concurrency:
25+
# Skip intermediate builds: always.
26+
# Cancel intermediate builds: only if it is a pull request build.
27+
group: ${{ github.workflow }}-${{ github.ref }}
28+
cancel-in-progress: ${{ startsWith(github.ref, 'refs/pull/') }}
29+
30+
jobs:
31+
integration:
32+
timeout-minutes: 45
33+
name: Integration Tests - ${{ matrix.package }} - Julia ${{ matrix.version }}
34+
runs-on: ${{ matrix.os }}
35+
container:
36+
image: ${{ (contains(matrix.os, 'linux') && 'ghcr.io/enzymead/reactant-docker-images@sha256:91e1edb7a7c869d5a70db06e417f22907be0e67ca86641d48adcea221fedc674' ) || '' }}
37+
env:
38+
JULIA_PKG_SERVER_REGISTRY_PREFERENCE: eager
39+
strategy:
40+
fail-fast: false
41+
matrix:
42+
version:
43+
- '1.10'
44+
- '1.11'
45+
os:
46+
- linux-x86-n2-32
47+
package:
48+
- Bijectors
49+
- DifferentiationInterface
50+
- DynamicExpressions
51+
steps:
52+
- uses: actions/checkout@v5
53+
- uses: julia-actions/setup-julia@v2
54+
with:
55+
version: ${{ matrix.version }}
56+
- uses: julia-actions/cache@v2
57+
- uses: julia-actions/julia-buildpkg@v1
58+
- name: "Install Dependencies"
59+
run: |
60+
julia --color=yes --project=test/integration/${{ matrix.package }} --threads=auto --check-bounds=yes -O1 -e 'using Pkg; Pkg.develop([PackageSpec(; path) for path in (".", "lib/EnzymeCore")])'
61+
shell: bash
62+
if: ${{ matrix.version == '1.10' }}
63+
- name: "Instantiate"
64+
run: |
65+
julia --color=yes --project=test/integration/${{ matrix.package }} --threads=auto --check-bounds=yes -O1 -e 'using Pkg; Pkg.instantiate()'
66+
shell: bash
67+
- name: "Run tests"
68+
run: |
69+
julia --color=yes --project=test/integration/${{ matrix.package }} --threads=auto --check-bounds=yes -O1 test/integration/${{ matrix.package }}/runtests.jl
70+
shell: bash

.gitignore

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -8,5 +8,5 @@ lib/EnzymeCore/Manifest.toml
88
/docs/Manifest.toml
99
/docs/build/
1010
/docs/src/generated/
11-
11+
test/integration/**/Manifest.toml
1212
.vscode

test/integration/Bijectors/Project.toml

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,14 @@
11
[deps]
22
Bijectors = "76274a88-744f-5084-9051-94815aaf08c4"
3+
Enzyme = "7da242da-08ed-463a-9acd-ee780be4f1d9"
4+
EnzymeCore = "f151be2c-9106-41f4-ab19-57ee4f262869"
35
FiniteDifferences = "26cc04aa-876d-5657-8c51-4c34ba976000"
46
StableRNGs = "860ef19b-820b-49d6-a774-d7a799459cd3"
57

8+
[sources]
9+
Enzyme = { path = "../../.." }
10+
EnzymeCore = { path = "../../../lib/EnzymeCore" }
11+
612
[compat]
713
Bijectors = "=0.13.16"
814
FiniteDifferences = "0.12.32"

test/integration/Bijectors/runtests.jl

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -131,7 +131,11 @@ end
131131
sum_b_binv_test_case(Bijectors.VecCholeskyBijector(:U), 3),
132132
sum_b_binv_test_case(Bijectors.VecCholeskyBijector(:U), 0),
133133
sum_b_binv_test_case(Bijectors.Coupling(Bijectors.Shift, Bijectors.PartitionMask(3, [1], [2])), 3),
134-
sum_b_binv_test_case(Bijectors.InvertibleBatchNorm(3), (3, 3)),
134+
sum_b_binv_test_case(
135+
Bijectors.InvertibleBatchNorm(3),
136+
(3, 3);
137+
runtime_activity = (VERSION >= v"1.11" ? Both : Neither)
138+
),
135139
sum_b_binv_test_case(Bijectors.LeakyReLU(0.2), 3),
136140
sum_b_binv_test_case(Bijectors.Logit(0.1, 0.3), 3),
137141
sum_b_binv_test_case(Bijectors.PDBijector(), (3, 3)),
@@ -186,6 +190,8 @@ end
186190
end,
187191
randn(rng, 7);
188192
name="PlanarLayer7",
193+
# https://github.com/TuringLang/Bijectors.jl/issues/415
194+
broken = (VERSION >= v"1.11" ? Forward : Neither),
189195
),
190196

191197
TestCase(
@@ -197,6 +203,7 @@ end
197203
end,
198204
randn(rng, 11);
199205
name="PlanarLayer11",
206+
broken = (VERSION >= v"1.11" ? Forward : Neither),
200207
),
201208
]
202209

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
[deps]
2+
ADTypes = "47edcb42-4c32-4615-8424-f2b9edc5f35b"
3+
DifferentiationInterface = "a0c0ee7d-e4b9-4e03-894e-1c5f64a51d63"
4+
DifferentiationInterfaceTest = "a82114a7-5aa3-49a8-9643-716bb13727a3"
5+
Enzyme = "7da242da-08ed-463a-9acd-ee780be4f1d9"
6+
EnzymeCore = "f151be2c-9106-41f4-ab19-57ee4f262869"
7+
StaticArrays = "90137ffa-7385-5640-81b9-e52037218182"
8+
Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40"
9+
10+
[sources]
11+
Enzyme = { path = "../../.." }
12+
EnzymeCore = { path = "../../../lib/EnzymeCore" }
13+
14+
[compat]
15+
ADTypes = "1.17.0"
16+
DifferentiationInterface = "=0.7.7"
17+
DifferentiationInterfaceTest = "=0.10.2"
18+
StaticArrays = "1.9"
Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,35 @@
1+
# DI integration tests for Enzyme
2+
3+
This folder contains tests to ensure that changes to Enzyme do not break integration with [DifferentiationInterface](https://github.com/JuliaDiff/DifferentiationInterface.jl) (DI).
4+
5+
## Relevant source files
6+
7+
The test utilities used here come from the sibling package [DifferentiationInterfaceTest](https://github.com/JuliaDiff/DifferentiationInterface.jl/tree/main/DifferentiationInterfaceTest) (DIT).
8+
Correctness checking itself is implemented in [`src/tests/correctness_eval.jl`](https://github.com/JuliaDiff/DifferentiationInterface.jl/blob/ed5655a90bf9f3a6092904070d353a9d705ebdc4/DifferentiationInterfaceTest/src/tests/correctness_eval.jl), which is where you will see test errors originating.
9+
Test scenarios are located in [`src/scenarios`](https://github.com/JuliaDiff/DifferentiationInterface.jl/tree/ed5655a90bf9f3a6092904070d353a9d705ebdc4/DifferentiationInterfaceTest/src/scenarios) (especially [`src/scenarios/default.jl`](https://github.com/JuliaDiff/DifferentiationInterface.jl/blob/ed5655a90bf9f3a6092904070d353a9d705ebdc4/DifferentiationInterfaceTest/src/scenarios/default.jl) and [`src/scenarios/modify.jl`](https://github.com/JuliaDiff/DifferentiationInterface.jl/blob/ed5655a90bf9f3a6092904070d353a9d705ebdc4/DifferentiationInterfaceTest/src/scenarios/modify.jl)) and in package extensions.
10+
The structure of a `Scenario` is defined in [`src/scenarios/scenario.jl`](https://github.com/JuliaDiff/DifferentiationInterface.jl/blob/ed5655a90bf9f3a6092904070d353a9d705ebdc4/DifferentiationInterfaceTest/src/scenarios/scenario.jl).
11+
12+
Scenario generation relies on internals of DifferentiationInterfaceTest, which is why its version is pinned in the `Project.toml`.
13+
14+
## Interpreting test errors
15+
16+
The most common test error you will see looks like
17+
18+
```julia
19+
Correctness: Test Failed at .../src/tests/correctness_eval.jl:...
20+
Expression: res1_out1_noval scen.res1
21+
```
22+
23+
Each test scenario `scen` contains a first-order result `res1` and a second-order result `res2`, which are the reference values we compare our autodiff results (i.e. the output of `DI.gradient` or `DI.jacobian`) against.
24+
The suffixes of the left-hand term are defined as follows:
25+
26+
- `_out` for the output of the operator, `_in` for the input if it is in-place
27+
- `1` for the first call to the operator, `2` for the second call (which is used to check that the preparation object has not been altered and can safely be reused)
28+
- `_val` if the operator also returns the value of the function (like `DI.value_and_gradient`), `_noval` otherwise
29+
30+
As you can see, several variants of each operator are tested, so a single bug will give rise to many different errors. In addition, different preparation mechanisms are also cycled through.
31+
The testset summary at the end of the CI log is probably the right place to start hunting down issues.
32+
33+
## What to do if a test fails
34+
35+
Open an issue on the DI repo with a link to the relevant PR or CI log.
Lines changed: 86 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,86 @@
1+
using ADTypes: AutoEnzyme
2+
using DifferentiationInterface: DifferentiationInterface
3+
using DifferentiationInterfaceTest:
4+
default_scenarios,
5+
sparse_scenarios,
6+
static_scenarios,
7+
test_differentiation,
8+
function_place,
9+
operator_place,
10+
FIRST_ORDER,
11+
SECOND_ORDER,
12+
Scenario
13+
using Enzyme: Enzyme
14+
using EnzymeCore: Forward, Reverse, Const, Duplicated
15+
using StaticArrays: StaticArrays
16+
using Test
17+
18+
logging = get(ENV, "CI", "false") == "false"
19+
20+
function remove_matrices(scens::Vector{<:Scenario}) # TODO: remove
21+
return filter(s -> s.x isa Union{Number, AbstractVector} && s.y isa Union{Number, AbstractVector}, scens)
22+
end
23+
24+
backends = [
25+
AutoEnzyme(; function_annotation = Const),
26+
AutoEnzyme(; mode = Forward),
27+
AutoEnzyme(; mode = Reverse),
28+
]
29+
30+
duplicated_backends = [
31+
AutoEnzyme(; mode = Forward, function_annotation = Duplicated),
32+
AutoEnzyme(; mode = Reverse, function_annotation = Duplicated),
33+
]
34+
35+
@testset verbose = true "DifferentiationInterface integration" begin
36+
test_differentiation(
37+
backends,
38+
default_scenarios(; include_constantified = true);
39+
excluded = SECOND_ORDER,
40+
logging,
41+
testset_name = "Generic first order",
42+
)
43+
44+
test_differentiation(
45+
backends[1],
46+
remove_matrices(default_scenarios(; include_constantified = true));
47+
excluded = FIRST_ORDER,
48+
logging,
49+
testset_name = "Generic second order",
50+
)
51+
52+
test_differentiation(
53+
backends[2],
54+
remove_matrices(
55+
default_scenarios(;
56+
include_normal = false,
57+
include_cachified = true,
58+
include_constantorcachified = true,
59+
use_tuples = true,
60+
)
61+
);
62+
excluded = SECOND_ORDER,
63+
logging,
64+
testset_name = "Caches",
65+
)
66+
67+
test_differentiation(
68+
duplicated_backends,
69+
remove_matrices(default_scenarios(; include_normal = false, include_closurified = true));
70+
excluded = SECOND_ORDER,
71+
logging,
72+
testset_name = "Closures",
73+
)
74+
75+
filtered_static_scenarios = filter(remove_matrices(static_scenarios())) do s
76+
operator_place(s) == :out && function_place(s) == :out
77+
end
78+
79+
test_differentiation(
80+
backends[2:3],
81+
filtered_static_scenarios;
82+
excluded = SECOND_ORDER,
83+
logging,
84+
testset_name = "Static arrays",
85+
)
86+
end
Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,11 @@
11
[deps]
22
DynamicExpressions = "a40a106e-89c9-4ca8-8020-a735e8728b6b"
3+
Enzyme = "7da242da-08ed-463a-9acd-ee780be4f1d9"
4+
EnzymeCore = "f151be2c-9106-41f4-ab19-57ee4f262869"
5+
6+
[sources]
7+
Enzyme = { path = "../../.." }
8+
EnzymeCore = { path = "../../../lib/EnzymeCore" }
39

410
[compat]
511
DynamicExpressions = "=0.18.5"

0 commit comments

Comments
 (0)