Add test framework support for running individual tests. (#310)

* Add test framework support for running individual tests.

* Add support for -ff to the test framework to fail fast without specifying an individual test to run.

Author: duckinator

Makefile

@@ -63,9 +63,6 @@ ifeq ($(VALGRIND),1)
 	override CFLAGS += -DUSE_VALGRIND
 	override TEST_CFLAGS += -DUSE_VALGRIND
 endif
-ifeq ($(TEST_FAIL_FAST),1)
-	override TEST_CFLAGS += -DTEST_FAIL_FAST=1
-endif
 
 CLEANLIBS = $(ARLIB)
 CLEANOBJS = $(OBJ_FILES)

test/framework/main.c

@@ -11,6 +11,16 @@
  */
 const size_t ARBITRARY_LENGTH_LIMIT = 500;
 
+int
+TEST_INIT_impl(TestState * test_state, const char *func, size_t line)
+{
+    if ((test_state->wanted_test != NULL) && (strcmp(test_state->wanted_test, func) != 0))
+        return 1;
+
+    printf("%s (line %li)\n", func, line);
+    return 0;
+}
+
 int
 TEST_BOUNDED_STRCMP(char *s1, char *s2)
 {
@@ -188,10 +198,27 @@ TEST_ASSERT_LIST_EQUAL_impl(TestState * test_state, char *actual_str, List *actu
 // implementation.
 static
 int
-test_run_impl(TestFn * tests[], TestCleanupFn * test_cleanup, bool use_mctx)
+test_run_impl(int argc, char *argv[], TestFn * tests[], TestCleanupFn * test_cleanup, bool use_mctx)
 {
 	TestState	test_state = {0};
 
+	bool fail_fast = false;
+
+	for (size_t i = 1; i < argc; i++)
+	{
+		// This loop only runs if an argument was passed.
+		// That argument will be -ff or a test name, and in both
+		// cases we want to fail fast.
+		fail_fast = true;
+
+		// If the argument isn't -ff, we assume it's a test name.
+		if (strncmp(argv[i], "-ff", 4) != 0) {
+			test_state.wanted_test = argv[i];
+			break;
+		}
+	}
+
+
 	pg_query_init();
 
 	for (size_t i = 0; tests[i] != NULL; i++)
@@ -207,10 +234,8 @@ test_run_impl(TestFn * tests[], TestCleanupFn * test_cleanup, bool use_mctx)
 		if (use_mctx)
 			pg_query_exit_memory_context(ctx);
 
-#ifdef TEST_FAIL_FAST
-		if (test_state.failed > 0)
+		if (fail_fast && test_state.failed > 0)
 			break;
-#endif
 	}
 
 	bool		failed = (test_state.failed > 0);
@@ -223,13 +248,13 @@ test_run_impl(TestFn * tests[], TestCleanupFn * test_cleanup, bool use_mctx)
 }
 
 int
-test_run(TestFn * tests[], TestCleanupFn * test_cleanup)
+test_run(int argc, char *argv[], TestFn * tests[], TestCleanupFn * test_cleanup)
 {
-	return test_run_impl(tests, test_cleanup, false);
+	return test_run_impl(argc, argv, tests, test_cleanup, false);
 }
 
 int
-test_run_with_mcxt(TestFn * tests[], TestCleanupFn * test_cleanup)
+test_run_with_mcxt(int argc, char *argv[], TestFn * tests[], TestCleanupFn * test_cleanup)
 {
-	return test_run_impl(tests, test_cleanup, true);
+	return test_run_impl(argc, argv, tests, test_cleanup, true);
 }

test/framework/main.h

@@ -40,6 +40,7 @@
 
 typedef struct
 {
+	char		*wanted_test;
 	size_t		passed;
 	size_t		failed;
 	size_t		skipped;
@@ -50,6 +51,7 @@ typedef void (TestFn) (TestState * test_state);
 typedef void (TestCleanupFn) (void);
 
 /* Functions defined in framework/main.c */
+int			TEST_INIT_impl(TestState * test_state, const char *func, size_t line);
 int			TEST_BOUNDED_STRCMP(char *s1, char *s2);
 void		TEST_ASSERT_NULL_impl(TestState * test_state, char *actual_str, void *actual);
 void		TEST_ASSERT_LIST_EQUAL_impl(TestState * test_state, char *actual_str, List *actual, List *expected);
@@ -63,7 +65,7 @@ void		TEST_ASSERT_STR_EQUAL_impl(TestState * test_state, char *actual_str, char
 #define TEST_SKIP(...) (printf("  SKIP: " __VA_ARGS__), test_state->skipped++)
 
 /* Prints the name of the test and the line number. */
-#define TEST_INIT() printf("%s (line %i)\n", __func__, __LINE__)
+#define TEST_INIT() do { if (TEST_INIT_impl(test_state, __func__, __LINE__) != 0) return; } while(0)
 
 /* Assert that `actual` is NULL. */
 #define TEST_ASSERT_NULL(actual) TEST_ASSERT_NULL_impl(test_state, #actual, actual)
@@ -79,8 +81,9 @@ void		TEST_ASSERT_STR_EQUAL_impl(TestState * test_state, char *actual_str, char
 
 /* Run each test in order, passing the same TestState object. */
 /* `test_cleanup()` is run between each test if it is not NULL. */
-int			test_run(TestFn * tests[], TestCleanupFn * test_cleanup);
-int			test_run_with_mcxt(TestFn * tests[], TestCleanupFn * test_cleanup);
+/* If argc is >1, the first argument is treated as a filter for tests. */
+int			test_run(int argc, char *argv[], TestFn * tests[], TestCleanupFn * test_cleanup);
+int			test_run_with_mcxt(int argc, char *argv[], TestFn * tests[], TestCleanupFn * test_cleanup);
 
 #define TEST_LIST_MAKE6(a,b,c,d,e,f) lappend(list_make5(a,b,c,d,e), f)
 #define TEST_LIST_MAKE7(a,b,c,d,e,f,g) lappend(TEST_LIST_MAKE6(a,b,c,d,e,f), g)

test/summary.c

@@ -253,5 +253,5 @@ main(int argc, char *argv[])
 		NULL
 	};
 
-	return test_run_with_mcxt(tests, &test_cleanup);
+	return test_run_with_mcxt(argc, argv, tests, &test_cleanup);
 }

test/summary_truncate.c

@@ -198,5 +198,5 @@ main(int argc, char *argv[])
 		NULL
 	};
 
-	return test_run_with_mcxt(tests, NULL);
+	return test_run_with_mcxt(argc, argv, tests, NULL);
 }