Skip to content

Commit eb3c804

Browse files
alliepiperalliepiper
committed
Add more test cases for stopping criterion parsing.
1 parent 0cd34ac commit eb3c804

File tree

1 file changed

+220
-14
lines changed

1 file changed

+220
-14
lines changed

‎testing/option_parser.cu‎

Lines changed: 220 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -1197,16 +1197,18 @@ void test_timeout()
11971197

11981198
void test_stopping_criterion()
11991199
{
1200-
{
1200+
{ // Per benchmark criterion
12011201
nvbench::option_parser parser;
1202-
parser.parse({"--benchmark",
1203-
"DummyBench",
1204-
"--stopping-criterion",
1205-
"entropy",
1206-
"--max-angle",
1207-
"0.42",
1208-
"--min-r2",
1209-
"0.6"});
1202+
parser.parse({
1203+
"--benchmark",
1204+
"DummyBench",
1205+
"--stopping-criterion",
1206+
"entropy",
1207+
"--max-angle",
1208+
"0.42",
1209+
"--min-r2",
1210+
"0.6",
1211+
});
12101212
const auto &states = parser_to_states(parser);
12111213

12121214
ASSERT(states.size() == 1);
@@ -1219,13 +1221,156 @@ void test_stopping_criterion()
12191221
ASSERT(criterion_params.get_float64("max-angle") == 0.42);
12201222
ASSERT(criterion_params.get_float64("min-r2") == 0.6);
12211223
}
1222-
{
1224+
{ // Global criterion
1225+
nvbench::option_parser parser;
1226+
parser.parse({
1227+
"--stopping-criterion",
1228+
"entropy",
1229+
"--max-angle",
1230+
"0.42",
1231+
"--min-r2",
1232+
"0.6",
1233+
"--benchmark",
1234+
"DummyBench",
1235+
});
1236+
const auto &states = parser_to_states(parser);
1237+
1238+
ASSERT(states.size() == 1);
1239+
ASSERT(states[0].get_stopping_criterion() == "entropy");
1240+
1241+
const nvbench::criterion_params &criterion_params = states[0].get_criterion_params();
1242+
ASSERT(criterion_params.has_value("max-angle"));
1243+
ASSERT(criterion_params.has_value("min-r2"));
1244+
1245+
ASSERT(criterion_params.get_float64("max-angle") == 0.42);
1246+
ASSERT(criterion_params.get_float64("min-r2") == 0.6);
1247+
}
1248+
{ // Global criterion, per-benchmark params
1249+
nvbench::option_parser parser;
1250+
parser.parse({
1251+
"--stopping-criterion",
1252+
"entropy",
1253+
"--benchmark",
1254+
"DummyBench",
1255+
"--max-angle",
1256+
"0.42",
1257+
"--min-r2",
1258+
"0.6",
1259+
});
1260+
const auto &states = parser_to_states(parser);
1261+
1262+
ASSERT(states.size() == 1);
1263+
ASSERT(states[0].get_stopping_criterion() == "entropy");
1264+
1265+
const nvbench::criterion_params &criterion_params = states[0].get_criterion_params();
1266+
ASSERT(criterion_params.has_value("max-angle"));
1267+
ASSERT(criterion_params.has_value("min-r2"));
1268+
1269+
ASSERT(criterion_params.get_float64("max-angle") == 0.42);
1270+
ASSERT(criterion_params.get_float64("min-r2") == 0.6);
1271+
}
1272+
{ // Global params to default criterion should work:
1273+
nvbench::option_parser parser;
1274+
parser.parse({
1275+
"--max-noise",
1276+
"0.5",
1277+
"--min-time",
1278+
"0.1",
1279+
"--benchmark",
1280+
"DummyBench",
1281+
"--stopping-criterion",
1282+
"entropy",
1283+
"--max-angle",
1284+
"0.42",
1285+
"--min-r2",
1286+
"0.6",
1287+
});
1288+
const auto &states = parser_to_states(parser);
1289+
1290+
ASSERT(states.size() == 1);
1291+
ASSERT(states[0].get_stopping_criterion() == "entropy");
1292+
1293+
const nvbench::criterion_params &criterion_params = states[0].get_criterion_params();
1294+
ASSERT(criterion_params.has_value("max-angle"));
1295+
ASSERT(criterion_params.has_value("min-r2"));
1296+
1297+
ASSERT(criterion_params.get_float64("max-angle") == 0.42);
1298+
ASSERT(criterion_params.get_float64("min-r2") == 0.6);
1299+
}
1300+
{ // Unknown stopping criterion should throw
1301+
bool exception_thrown = false;
1302+
try
1303+
{
1304+
nvbench::option_parser parser;
1305+
parser.parse({
1306+
"--benchmark",
1307+
"DummyBench",
1308+
"--stopping-criterion",
1309+
"I_do_not_exist",
1310+
});
1311+
}
1312+
catch (const std::runtime_error &)
1313+
{
1314+
exception_thrown = true;
1315+
}
1316+
ASSERT(exception_thrown);
1317+
}
1318+
{ // Global criterion to non-default params without global --stopping-criterion should throw
1319+
bool exception_thrown = false;
12231320
try
12241321
{
12251322
nvbench::option_parser parser;
12261323
parser.parse({
12271324
"--max-angle",
12281325
"0.42",
1326+
"--min-r2",
1327+
"0.6",
1328+
"--benchmark",
1329+
"DummyBench",
1330+
"--stopping-criterion",
1331+
"entropy",
1332+
});
1333+
}
1334+
catch (const std::runtime_error &)
1335+
{
1336+
exception_thrown = true;
1337+
}
1338+
ASSERT(exception_thrown);
1339+
}
1340+
{ // Invalid global param throws exception:
1341+
bool exception_thrown = false;
1342+
try
1343+
{
1344+
nvbench::option_parser parser;
1345+
parser.parse({
1346+
"--max-angle",
1347+
"0.42",
1348+
"--benchmark",
1349+
"DummyBench",
1350+
"--stopping-criterion",
1351+
"entropy",
1352+
"--min-r2",
1353+
"0.6",
1354+
"--max-angle",
1355+
"0.42",
1356+
"--benchmark",
1357+
"TestBench",
1358+
"--stopping-criterion",
1359+
"stdrel",
1360+
});
1361+
}
1362+
catch (const std::runtime_error & /*ex*/)
1363+
{
1364+
exception_thrown = true;
1365+
}
1366+
ASSERT(exception_thrown);
1367+
}
1368+
{ // Invalid per-bench param throws exception:
1369+
bool exception_thrown = false;
1370+
try
1371+
{
1372+
nvbench::option_parser parser;
1373+
parser.parse({
12291374
"--benchmark",
12301375
"DummyBench",
12311376
"--stopping-criterion",
@@ -1238,14 +1383,75 @@ void test_stopping_criterion()
12381383
"TestBench",
12391384
"--stopping-criterion",
12401385
"stdrel",
1386+
"--max-angle",
1387+
"0.42",
1388+
});
1389+
}
1390+
catch (const std::runtime_error & /*ex*/)
1391+
{
1392+
exception_thrown = true;
1393+
}
1394+
ASSERT(exception_thrown);
1395+
}
1396+
{ // global param-before-criterion throws exception:
1397+
bool exception_thrown = false;
1398+
try
1399+
{
1400+
nvbench::option_parser parser;
1401+
parser.parse({
1402+
"--min-r2", //
1403+
"0.6",
1404+
"--stopping-criterion",
1405+
"entropy",
1406+
"--benchmark",
1407+
"DummyBench",
1408+
});
1409+
}
1410+
catch (const std::runtime_error & /*ex*/)
1411+
{
1412+
exception_thrown = true;
1413+
}
1414+
ASSERT(exception_thrown);
1415+
}
1416+
{ // per-benchmark param-before-criterion throws exception:
1417+
bool exception_thrown = false;
1418+
try
1419+
{
1420+
nvbench::option_parser parser;
1421+
parser.parse({
1422+
"--benchmark", //
1423+
"DummyBench",
1424+
"--min-r2",
1425+
"0.6",
1426+
"--stopping-criterion",
1427+
"entropy",
12411428
});
1242-
ASSERT(false);
12431429
}
12441430
catch (const std::runtime_error & /*ex*/)
12451431
{
1246-
// `max-angle` isn't applicable to `stdrel`
1247-
// fmt::print(stderr, "{}", ex.what());
1432+
exception_thrown = true;
1433+
}
1434+
ASSERT(exception_thrown);
1435+
}
1436+
{ // Invalid param type throws exception:
1437+
bool exception_thrown = false;
1438+
try
1439+
{
1440+
nvbench::option_parser parser;
1441+
parser.parse({
1442+
"--benchmark", //
1443+
"DummyBench",
1444+
"--stopping-criterion",
1445+
"entropy",
1446+
"--min-r2",
1447+
"\"foo\"",
1448+
});
1449+
}
1450+
catch (const std::runtime_error &)
1451+
{
1452+
exception_thrown = true;
12481453
}
1454+
ASSERT(exception_thrown);
12491455
}
12501456
}
12511457

@@ -1291,6 +1497,6 @@ try
12911497
}
12921498
catch (std::exception &err)
12931499
{
1294-
fmt::print(stderr, "{}", err.what());
1500+
fmt::print(stderr, "Unexpected exception:\n{}\n", err.what());
12951501
return 1;
12961502
}

0 commit comments

Comments
 (0)