make parameterized testing easier

[ayjayt]

I use parameterization in testing because it finds a lot of corner-case bugs.

Test results look like this:

I test many possible new configurations including extremes, etc, and then also test a range of configurations that shouldn't impact the new feature but might because of code proximity.

It does find bugs

In Jasmine and w/ ES5+Promises it's hard to implement manually because of all the weird scoping rules and promises and lack of ES6 tools like let. You can see my implementation here:

plotly.js/test/jasmine/tests/pikul_test.js

Lines 28 to 143 in fe6f8d0

	/* ****** PARAMETERIZATION ******* */
	/* TICK CONFIG GENERATORS */
	var MAJOR = 10; // `ticklen:10`
	var MINOR = 5; // `ticklen:5`
	// ticksOff() generates a config for no ticks
	function ticksOff() { return {ticklen: 0, showticklabels: false, nticks: 0}; }
	// generateTickConfig() can generate randomized but valid configs for `tickmode` "domain array" and "full domain"
	function generateTickConfig(tickLen, tickmode, nticks) {
	if(tickmode === undefined) tickmode = 'domain array'; // default
	var standardConfig = {tickmode: tickmode, ticklen: tickLen, showticklabels: false}; // no labels!
	// We analyze DOM to find number and position of ticks, labels make it harder.
	// Tick values will be random:
	if(tickmode === 'domain array') { // 'domain array' will have random tick proportions
	var n = Math.floor(Math.random() * 100);
	var tickVals = [];
	for(var i = 0; i <= n; i++) {
	// NOTE: MEANT TO BE DIFFERENT EVERYTIME
	var intermediate = (Math.trunc(Math.random() * 150) - 25) / 100; // Number between -.25 and 1.25 w/ 2 decimals max
	tickVals.push(Math.min(Math.max(intermediate, 0), 1)); // 2 decimal number between 0 and 1 w/ higher odds of 0 or 1
	}
	standardConfig.tickvals = tickVals;
	} else if(tickmode === 'full domain') { // TODO: full domain _could_ have a random number of ticks
	standardConfig.nticks = nticks;
	}
	return standardConfig;
	}
	// areTicks() returns true if `config`, an axis config, contains ticks: false otherwise.
	function areTicks(config) { // Check if ticks exists in a generated config
	return (config !== undefined && config.ticklen !== undefined && config.ticklen !== 0);
	}
	/* LOOP THROUGH ALL POSSIBLE COMBINATIONS:
	* xAxis major has ticks, xAxis minor has ticks, yAxis major does not, yAxis minor does, etc */
	// numbers 0 through 15 are all possible combination of 4 boolean values (0001, 0010, 0011, 0100, 0101, etc)
	var XMAJOR = 1;// 0b0001;
	var XMINOR = 2;// 0b0010;
	var YMAJOR = 4;// 0b0100;
	var YMINOR = 8;// 0b1000;
	// binaryToTickType converts binary to info string
	function binaryToTickType(bin) {
	var str = [];
	if(bin & XMAJOR) str.push('xMajor');
	if(bin & XMINOR) str.push('xMinor');
	if(bin & YMAJOR) str.push('yMajor');
	if(bin & YMINOR) str.push('yMinor');
	if(str.length) {
	return str.join(', ');
	}
	return 'None';
	}
	/* PARAMETERIZE POSSIBLE TYPES OF GRAPH */
	var graphTypes = [
	{ type: 'linear' },
	{ type: 'log'},
	{ type: 'date'},
	{ type: 'category'},
	];
	/* getParameters() will loop through all possible parameters, initializing it the first time, and return false the last */
	/* it's for for-loops */
	function getParameters(op) {
	// Initializize
	if(op === undefined) return {tickConfig: 0, graphTypeIndex: 0};
	// Loop through 15 possible tickConfigs
	if(++op.tickConfig > 15) op.tickConfig = 0;
	else return op;
	// Loop through 4 graph types after each full loop above
	if(++op.graphTypeIndex >= graphTypes.length) return false;
	return op;
	}
	// Loops MUST be outside tests do to scopes (and better for output, honestly)
	for(var parameters = getParameters(); parameters; parameters = getParameters(parameters)) {
	// Give parameters there own variable
	var xGraphType = graphTypes[parameters.graphTypeIndex];
	var tickConfig = parameters.tickConfig;
	// Linters don't like variable redeclaration in subscope so make all testing same scope
	var paramInfo = 'on axes ' + binaryToTickType(tickConfig) + ' for graph type: ' + xGraphType.type;
	var xConfig = (tickConfig & XMAJOR) ? generateTickConfig(MAJOR) : ticksOff(); // generate configs
	xConfig.minor = (tickConfig & XMINOR) ? generateTickConfig(MINOR) : ticksOff();
	var yConfig = (tickConfig & YMAJOR) ? generateTickConfig(MAJOR) : ticksOff();
	yConfig.minor = (tickConfig & YMINOR) ? generateTickConfig(MINOR) : ticksOff();
	// Configs are random, so we should inspect if test fails:
	var configInfo = '';
	configInfo += areTicks(xConfig) ? '\n ' + 'xMajor: ' + makeSet(xConfig.tickvals).length + ' unique vals' : '';
	configInfo += areTicks(xConfig.minor) ? '\n ' + 'xMinor: ' + makeSet(xConfig.minor.tickvals).length + ' unique vals' : '';
	configInfo += areTicks(yConfig) ? '\n ' + 'yMajor: ' + makeSet(yConfig.tickvals).length + ' unique vals' : '';
	configInfo += areTicks(yConfig.minor) ? '\n ' + 'yMinor: ' + makeSet(yConfig.minor.tickvals).length + ' unique vals' : '';
	// variablesToInject + closure function(scopeLock) is a necessary result of using a version w promises but w/o `let`
	var variablesToInject = {
	xConfig: xConfig, // Generated xConfig
	yConfig: yConfig, // Generated yConfig
	xGraphType: xGraphType, // graphType parameter
	tickConfig: tickConfig, // tickConfig parameter
	paramInfo: paramInfo, // info string
	configInfo: configInfo // info string
	};
	(function(scopeLock) {
	describe('`tickmode`:"domain array"', function() {

The messiness comes from scoping rules and reusing logic and etc. But it works well!

There is a module available on GitHub that has a much nicer interface: https://github.com/paucls/jasmine-parameterized
but not sure about maintenance or other solutions.

Thoughts?

edit: also i'm back after a move, will get back around to this stuff mid/late next week- have to prepare a course