Skip to content

Commit d5a5d8f

Browse files
PetroZarytskyivgvassilev
PetroZarytskyi
authored and
vgvassilev
committed
Test error estimation numerically
Currently, we only check the code produced in error estimation. The produced errors are never checked. This makes it hard to understand whether certain changes break error estimation or not. This PR introduces numerical checks for all error estimation tests. One of the reasons errors were not tested was that they are usually of order `10^-7`. This is because all errors get multiplied by `std::numeric_limits<float>::epsilon()`, which is equal to `2^-23 = 1.19e-7`. This makes it inconvenient to compare the results with theoretical predictions. This PR adds `TEST_ERROR_ESTIMATION` that normalizes the error by `std::numeric_limits<float>::epsilon()` before printing it. Since dividing by such small numbers could impact precission, the normalization is done as `std::ldexp(error, std::numeric_limits<PrecissionType>::digits - 1)`. `std::ldexp` lets us change the exponent of the float directly, and multiply the number by `2^23` without changing the mantissa. `PrecissionType` is templated so that we can test other floating types in the future.
1 parent 541b063 commit d5a5d8f

6 files changed

Lines changed: 148 additions & 48 deletions

File tree

‎test/ErrorEstimation/Assignments.C‎

Lines changed: 29 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,9 @@
11
// RUN: %cladclang -I%S/../../include -oAssignments.out %s 2>&1 | %filecheck %s
2-
// RUN: ./Assignments.out
2+
// RUN: ./Assignments.out | %filecheck_exec %s
3+
// XFAIL: valgrind
34

45
#include "clad/Differentiator/Differentiator.h"
6+
#include "../TestUtils.h"
57

68
#include <cmath>
79

@@ -158,12 +160,30 @@ float func8(int x, int y) {
158160
//CHECK-NEXT: }
159161

160162
int main() {
161-
clad::estimate_error(func);
162-
clad::estimate_error(func2);
163-
clad::estimate_error(func3);
164-
clad::estimate_error(func4);
165-
clad::estimate_error(func5);
166-
clad::estimate_error(func6);
167-
clad::estimate_error(func7);
168-
clad::estimate_error(func8);
163+
int dxi = 0, dyi = 0;
164+
float dxf = 0, dyf = 0;
165+
166+
INIT_ERROR_ESTIMATION(func);
167+
TEST_ERROR_ESTIMATION(func, /*PrecissionType*/float, 3, 5, &dxf, &dyf); // CHECK-EXEC: {16.00}
168+
169+
INIT_ERROR_ESTIMATION(func2);
170+
TEST_ERROR_ESTIMATION(func2, /*PrecissionType*/float, -2, 2, &dxf, &dyi); // CHECK-EXEC: {4.00}
171+
172+
INIT_ERROR_ESTIMATION(func3);
173+
TEST_ERROR_ESTIMATION(func3, /*PrecissionType*/float, 8, 4, &dyi, &dyi); // CHECK-EXEC: {0.00}
174+
175+
INIT_ERROR_ESTIMATION(func4);
176+
TEST_ERROR_ESTIMATION(func4, /*PrecissionType*/float, 0, 9, &dxf, &dyf); // CHECK-EXEC: {36.00}
177+
178+
INIT_ERROR_ESTIMATION(func5);
179+
TEST_ERROR_ESTIMATION(func5, /*PrecissionType*/float, 5, -7, &dxf, &dyf); // CHECK-EXEC: {56.00}
180+
181+
INIT_ERROR_ESTIMATION(func6);
182+
TEST_ERROR_ESTIMATION(func6, /*PrecissionType*/float, 3, &dxf); // CHECK-EXEC: {3.00}
183+
184+
INIT_ERROR_ESTIMATION(func7);
185+
TEST_ERROR_ESTIMATION(func7, /*PrecissionType*/float, 2, 1, &dxf, &dyf); // CHECK-EXEC: {6.00}
186+
187+
INIT_ERROR_ESTIMATION(func8);
188+
TEST_ERROR_ESTIMATION(func8, /*PrecissionType*/float, -1, 6, &dyi, &dyi); // CHECK-EXEC: {0.00}
169189
}

‎test/ErrorEstimation/BasicOps.C‎

Lines changed: 36 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,9 @@
1-
// RUN: %cladclang %s -I%S/../../include -fsyntax-only -Xclang -verify 2>&1 | %filecheck %s
1+
// RUN: %cladclang %s -I%S/../../include -Xclang -verify -oBasicOps.out 2>&1 | %filecheck %s
2+
// RUN: ./BasicOps.out | %filecheck_exec %s
3+
// XFAIL: valgrind
24

35
#include "clad/Differentiator/Differentiator.h"
6+
#include "../TestUtils.h"
47

58
#include <cmath>
69

@@ -328,14 +331,36 @@ double func10(double x, double y) {
328331
// CHECK-NEXT: }
329332

330333
int main() {
331-
clad::estimate_error(func);
332-
clad::estimate_error(func2);
333-
clad::estimate_error(func3);
334-
clad::estimate_error(func4);
335-
clad::estimate_error(func5);
336-
clad::estimate_error(func6);
337-
clad::estimate_error(func7);
338-
clad::estimate_error(func8);
339-
clad::estimate_error(func9);
340-
clad::estimate_error(func10);
334+
float dx = 0, dy = 0;
335+
double dxd = 0, dyd = 0;
336+
337+
INIT_ERROR_ESTIMATION(func);
338+
TEST_ERROR_ESTIMATION(func, /*PrecissionType*/float, 3, 1, &dx, &dy); // CHECK-EXEC: {76.00}
339+
340+
INIT_ERROR_ESTIMATION(func2);
341+
TEST_ERROR_ESTIMATION(func2, /*PrecissionType*/float, 4, 9, &dx, &dy); // CHECK-EXEC: {0.32}
342+
343+
INIT_ERROR_ESTIMATION(func3);
344+
TEST_ERROR_ESTIMATION(func3, /*PrecissionType*/float, 5, 2, &dx, &dy); // CHECK-EXEC: {136.00}
345+
346+
INIT_ERROR_ESTIMATION(func4);
347+
TEST_ERROR_ESTIMATION(func4, /*PrecissionType*/float, 2, 4, &dx, &dy); // CHECK-EXEC: {124.36}
348+
349+
INIT_ERROR_ESTIMATION(func5);
350+
TEST_ERROR_ESTIMATION(func5, /*PrecissionType*/float, 1, 4, &dx, &dy); // CHECK-EXEC: {3.03}
351+
352+
INIT_ERROR_ESTIMATION(func6);
353+
TEST_ERROR_ESTIMATION(func6, /*PrecissionType*/float, 6, -1, &dx, &dy); // CHECK-EXEC: {330.00}
354+
355+
INIT_ERROR_ESTIMATION(func7);
356+
TEST_ERROR_ESTIMATION(func7, /*PrecissionType*/float, 7, &dx); // CHECK-EXEC: {14.00}
357+
358+
INIT_ERROR_ESTIMATION(func8);
359+
TEST_ERROR_ESTIMATION(func8, /*PrecissionType*/float, -3, 2, &dx, &dy); // CHECK-EXEC: {47.00}
360+
361+
INIT_ERROR_ESTIMATION(func9);
362+
TEST_ERROR_ESTIMATION(func9, /*PrecissionType*/float, 0, 5, &dx, &dy); // CHECK-EXEC: {25.00}
363+
364+
INIT_ERROR_ESTIMATION(func10);
365+
TEST_ERROR_ESTIMATION(func10, /*PrecissionType*/float, 8, 5, &dxd, &dyd); // CHECK-EXEC: {240.00}
341366
}

‎test/ErrorEstimation/ConditonalStatements.C‎

Lines changed: 17 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,9 @@
11
// RUN: %cladclang -I%S/../../include -oCondStmts.out %s 2>&1 | %filecheck %s
2+
// RUN: ./CondStmts.out | %filecheck_exec %s
3+
// XFAIL: valgrind
24

35
#include "clad/Differentiator/Differentiator.h"
6+
#include "../TestUtils.h"
47

58
#include <cmath>
69

@@ -175,8 +178,18 @@ float func4(float x, float y) {
175178
//CHECK-NEXT: }
176179

177180
int main() {
178-
clad::estimate_error(func);
179-
clad::estimate_error(func2);
180-
clad::estimate_error(func3);
181-
clad::estimate_error(func4);
181+
float dx = 0, dy = 0;
182+
183+
INIT_ERROR_ESTIMATION(func);
184+
TEST_ERROR_ESTIMATION(func, /*PrecissionType*/float, 8, 2, &dx, &dy); // CHECK-EXEC: {80.00}
185+
186+
INIT_ERROR_ESTIMATION(func2);
187+
TEST_ERROR_ESTIMATION(func2, /*PrecissionType*/float, 2, &dx); // CHECK-EXEC: {12.00}
188+
189+
INIT_ERROR_ESTIMATION(func3);
190+
TEST_ERROR_ESTIMATION(func3, /*PrecissionType*/float, 1, 3, &dx, &dy); // CHECK-EXEC: {8.00}
191+
192+
INIT_ERROR_ESTIMATION(func4);
193+
TEST_ERROR_ESTIMATION(func4, /*PrecissionType*/float, -7, 2, &dx, &dy); // CHECK-EXEC: {0.20}
194+
182195
}

‎test/ErrorEstimation/LoopsAndArrays.C‎

Lines changed: 29 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,9 @@
11
// RUN: %cladclang -I%S/../../include -oLoopsAndArrays.out %s 2>&1 | %filecheck %s
2+
// RUN: ./LoopsAndArrays.out | %filecheck_exec %s
23
// XFAIL: valgrind
34

45
#include "clad/Differentiator/Differentiator.h"
6+
#include "../TestUtils.h"
57

68
#include <cmath>
79

@@ -44,7 +46,6 @@ float func(float* p, int n) {
4446
//CHECK-NEXT: _final_error += std::abs(_d_p[i0] * p[i0] * {{.+}});
4547
//CHECK-NEXT: }
4648

47-
4849
float func2(float x) {
4950
float z;
5051
for (int i = 0; i < 9; i++) {
@@ -329,10 +330,31 @@ double func6(double x) {
329330
//CHECK-NEXT: }
330331

331332
int main() {
332-
clad::estimate_error(func);
333-
clad::estimate_error(func2);
334-
clad::estimate_error(func3);
335-
clad::estimate_error(func4);
336-
clad::estimate_error(func5);
337-
clad::estimate_error(func6);
333+
float p[] = {1, 2, 3, 4, 5}, dp[5] = {0};
334+
int dn = 0;
335+
INIT_ERROR_ESTIMATION(func);
336+
TEST_ERROR_ESTIMATION(func, /*PrecissionType*/float, p, 5, dp, &dn); // CHECK-EXEC: {50.00}
337+
338+
float dx = 0, dy = 0;;
339+
INIT_ERROR_ESTIMATION(func2);
340+
TEST_ERROR_ESTIMATION(func2, /*PrecissionType*/float, 3, &dx); // CHECK-EXEC: {72.00}
341+
342+
INIT_ERROR_ESTIMATION(func3);
343+
TEST_ERROR_ESTIMATION(func3, /*PrecissionType*/float, 2, 6, &dx, &dy); // CHECK-EXEC: {40.00}
344+
345+
float x_arr[10] = {1, 3, 5, 7, 9, 11, 13, 15, 17, 19};
346+
float y_arr[10] = {0, -2, -4, -6, -8, -10, -12, -14, -16, -18};
347+
float dx_arr[10] = {0}, dy_arr[10] = {0};
348+
INIT_ERROR_ESTIMATION(func4);
349+
TEST_ERROR_ESTIMATION(func4, /*PrecissionType*/float, x_arr, y_arr, dx_arr, dy_arr); // CHECK-EXEC: {255.00}
350+
351+
double x_arr_d[3] = {1, 3, 5}, dx_arr_d[3] = {0};
352+
double y_arr_d[3] = {0, -2, -4}, dy_arr_d[3] = {0};
353+
double out[3] = {0}, dout[3] = {0};
354+
INIT_ERROR_ESTIMATION(func5);
355+
TEST_ERROR_ESTIMATION(func5, /*PrecissionType*/float, x_arr_d, y_arr_d, out, dx_arr_d, dy_arr_d, dout); // CHECK-EXEC: {48.00}
356+
357+
double dx_d = 0;
358+
INIT_ERROR_ESTIMATION(func6);
359+
TEST_ERROR_ESTIMATION(func6, /*PrecissionType*/float, -3, &dx_d); // CHECK-EXEC: {405.00}
338360
}

‎test/ErrorEstimation/LoopsAndArraysExec.C‎

Lines changed: 15 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@
33
// XFAIL: valgrind
44

55
#include "clad/Differentiator/Differentiator.h"
6+
#include "../TestUtils.h"
67

78
#include <cmath>
89

@@ -155,32 +156,29 @@ double divSum(float* a, float* b, int n) {
155156
//CHECK-NEXT: }
156157

157158
int main() {
158-
auto df = clad::estimate_error(runningSum);
159159
float arrf[3] = {0.456, 0.77, 0.95};
160-
double finalError = 0;
161160
float darr[3] = {0, 0, 0};
162161
int dn = 0;
163-
df.execute(arrf, 3, darr, &dn, finalError);
164-
printf("Result (RS) = {%.2f, %.2f, %.2f} error = %.5f\n", darr[0], darr[1],
165-
darr[2], finalError); // CHECK-EXEC: Result (RS) = {1.00, 2.00, 1.00} error = 0.00000
162+
INIT_ERROR_ESTIMATION(runningSum);
163+
TEST_ERROR_ESTIMATION(runningSum, /*PrecissionType*/float, arrf, 3, darr, &dn); // CHECK-EXEC: {7.12}
164+
printf("Result (RS) = {%.2f, %.2f, %.2f}\n", darr[0], darr[1], darr[2]);
165+
// CHECK-EXEC: Result (RS) = {1.00, 2.00, 1.00}
166166

167-
finalError = 0;
168167
darr[0] = darr[1] = darr[2] = 0;
169168
dn = 0;
170169
float darr2[3] = {0, 0, 0};
171-
auto df2 = clad::estimate_error(mulSum);
172-
df2.execute(arrf, arrf, 3, darr, darr2, &dn, finalError);
173-
printf("Result (MS) = {%.2f, %.2f, %.2f}, {%.2f, %.2f, %.2f} error = %.5f\n",
174-
darr[0], darr[1], darr[2], darr2[0], darr2[1], darr2[2],
175-
finalError); // CHECK-EXEC: Result (MS) = {2.18, 2.18, 2.18}, {2.18, 2.18, 2.18} error = 0.00000
170+
INIT_ERROR_ESTIMATION(mulSum);
171+
TEST_ERROR_ESTIMATION(mulSum, /*PrecissionType*/float, arrf, arrf, 3, darr, darr2, &dn); // CHECK-EXEC: {28.85}
172+
printf("Result (MS) = {%.2f, %.2f, %.2f}, {%.2f, %.2f, %.2f}\n",
173+
darr[0], darr[1], darr[2], darr2[0], darr2[1], darr2[2]);
174+
// CHECK-EXEC: Result (MS) = {2.18, 2.18, 2.18}, {2.18, 2.18, 2.18}
176175

177-
finalError = 0;
178176
darr[0] = darr[1] = darr[2] = 0;
179177
darr2[0] = darr2[1] = darr2[2] = 0;
180178
dn = 0;
181-
auto df3 = clad::estimate_error(divSum);
182-
df3.execute(arrf, arrf, 3, darr, darr2, &dn, finalError);
183-
printf("Result (DS) = {%.2f, %.2f, %.2f}, {%.2f, %.2f, %.2f} error = %.5f\n",
184-
darr[0], darr[1], darr[2], darr2[0], darr2[1], darr2[2],
185-
finalError); // CHECK-EXEC: Result (DS) = {2.19, 1.30, 1.05}, {-2.19, -1.30, -1.05} error = 0.00000
179+
INIT_ERROR_ESTIMATION(divSum);
180+
TEST_ERROR_ESTIMATION(divSum, /*PrecissionType*/float, arrf, arrf, 3, darr, darr2, &dn); // CHECK-EXEC: {12.00}
181+
printf("Result (DS) = {%.2f, %.2f, %.2f}, {%.2f, %.2f, %.2f}\n",
182+
darr[0], darr[1], darr[2], darr2[0], darr2[1], darr2[2]);
183+
// CHECK-EXEC: Result (DS) = {2.19, 1.30, 1.05}, {-2.19, -1.30, -1.05}
186184
}

‎test/TestUtils.h‎

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -115,6 +115,14 @@ void run_gradient_impl(CF cf, index_pack<S...> s, Args&&... args) {
115115
cf.execute(args...);
116116
display(std::get<S>(t)...);
117117
}
118+
template <typename PrecissionType, class CF, std::size_t... S, class... Args>
119+
void run_error_estimation_impl(CF cf, index_pack<S...> s, Args&&... args) {
120+
std::tuple<Args...> t = {args...};
121+
reset(std::get<S>(t)...);
122+
double error = 0;
123+
cf.execute(args..., error);
124+
display(std::ldexp(error, std::numeric_limits<PrecissionType>::digits - 1));
125+
}
118126

119127
template <class CF, class... Args>
120128
void run_jacobian_impl(CF cf, std::size_t size, Args&&... args) {
@@ -156,6 +164,14 @@ void run_gradient(CF cf, Args&&... args) {
156164
run_gradient_impl(cf, DerivativeArgsRange(), std::forward<Args>(args)...);
157165
}
158166

167+
template <typename PrecissionType, class CF, class... Args>
168+
void run_error_estimation(CF cf, Args&&... args) {
169+
using DerivativeArgsRange =
170+
typename GenerateRange<sizeof...(Args) / 2,
171+
sizeof...(Args) - 1>::type;
172+
run_error_estimation_impl<PrecissionType>(cf, DerivativeArgsRange(), std::forward<Args>(args)...);
173+
}
174+
159175
template <std::size_t NumOfDerivativeArgs, std::size_t size, class CF,
160176
class... Args>
161177
void run_jacobian(CF cf, Args&&... args) {
@@ -227,6 +243,9 @@ void EssentiallyEqualArrays(A* a, B* b, unsigned size) {
227243
GET_MACRO(__VA_ARGS__, INIT_HESSIAN_SPECIFIC, INIT_HESSIAN_ALL) \
228244
(__VA_ARGS__)
229245

246+
#define INIT_ERROR_ESTIMATION(fn) \
247+
auto fn##_err = clad::estimate_error(fn);
248+
230249
#define TEST_GRADIENT(fn, numOfDerivativeArgs, ...) \
231250
test_utils::run_gradient<numOfDerivativeArgs>(fn##_grad, __VA_ARGS__);
232251

@@ -239,5 +258,8 @@ void EssentiallyEqualArrays(A* a, B* b, unsigned size) {
239258
#define TEST_HESSIAN(fn, numOfDerivativeArgs, ...) \
240259
test_utils::run_hessian<numOfDerivativeArgs>(fn##_hessian, __VA_ARGS__);
241260

261+
#define TEST_ERROR_ESTIMATION(fn, PrecissionType, ...) \
262+
test_utils::run_error_estimation<PrecissionType>(fn##_err, __VA_ARGS__);
263+
242264
#endif
243265
}

0 commit comments

Comments
 (0)