Add a sample project to test selectors

.gitignore

@@ -10,3 +10,9 @@ pom.xml.asc
 .hgignore
 .hg/
 /test-results/
+dev-resources/weasley/target/
+dev-resources/weasley/.lein-failures
+dev-resources/weasley/test-results/
+dev-resources/weasley/.lein-repl-history
+test_out.txt
+dev-resources/weasley/test_out.txt

Makefile

@@ -0,0 +1,19 @@
+weasley-test: root-test
+	@echo "------------------------------------"
+	@echo "Testing internal project Weasley"
+	@echo "------------------------------------"
+
+	# The goal is to compare test runs with Leiningen where
+	# possible (is there a better way than this?)
+	sh test.sh
+
+root-test:
+	@echo "------------------------------------"
+	@echo "Testing root project circleci.test"
+	@echo "------------------------------------"
+	lein do clean, compile
+	lein test
+
+test: root-test
+
+.PHONY: test

dev-resources/weasley/circleci.test.version

@@ -0,0 +1 @@
+(def circleci-test-version "0.4.3")

dev-resources/weasley/dev-resources/circleci_test/config.clj

@@ -0,0 +1,24 @@
+(require '[circleci.test.report :refer (clojure-test-reporter)])
+(require '[circleci.test.report.junit :as junit])
+(require '[clojure.java.io :as io])
+
+(def ^:dynamic *global-counter* 0)
+
+{:selectors {:all (constantly true)
+             :default (complement :failing)
+             :select-vars (fn [m]
+                            (.endsWith (str (:name m)) "-2"))
+             :combination :combination}
+ :test-results-dir (or (System/getenv "CIRCLE_TEST_REPORTS")
+                       "test-results")
+ :reporters [clojure-test-reporter junit/reporter]
+ :global-fixture (fn [f]
+                   (try
+                     (io/delete-file "global_fixture_test.out" true)
+                     (assert (zero? *global-counter*))
+                     (binding [*global-counter* (inc *global-counter*)]
+                       (spit "global_fixture_test.out" *global-counter* :append true)
+                       (f))
+                     (finally
+                       (io/delete-file "global_fixture_test.out" true)
+                       (assert (false? (.exists (clojure.java.io/file "global_fixture_test.out")))))))}

dev-resources/weasley/project.clj

@@ -0,0 +1,11 @@
+(load-file "./circleci.test.version")
+(defproject circleci/weasley "0.1.0-SNAPSHOT"
+  :dependencies [[org.clojure/clojure "1.10.1"]
+                 [circleci/circleci.test ~circleci-test-version]]
+  :target-path "target/%s"
+  :profiles {:uberjar {:aot :all}}
+  :main weasley.core
+  :aot [weasley.core]
+  :aliases {"test" ["run" "-m" "circleci.test/dir" :project/test-paths]
+            "tests" ["run" "-m" "circleci.test"]
+            "retest" ["run" "-m" "circleci.test.retest"]})

dev-resources/weasley/src/weasley/core.clj

@@ -0,0 +1,7 @@
+(ns weasley.core
+  (:gen-class))
+
+(defn -main
+  "I don't do a whole lot ... yet."
+  [& args]
+  (println "Hello, World!"))

dev-resources/weasley/test/weasley/sample_test_ns1.clj

@@ -0,0 +1,53 @@
+(ns weasley.sample-test-ns1
+  (:require [clojure.test :refer :all]))
+
+(def ^:dynamic vonce 0)
+(def ^:dynamic veach 0)
+(def ^:dynamic vcommon 0)
+
+(deftest test-1
+  (println "Running: circleci.sample-test-ns1/test-1")
+  (is (= 1 1))
+  (is (zero? veach))
+  (is (zero? vonce))
+  (is (zero? vcommon)))
+
+(deftest test-2
+  (println "Running: circleci.sample-test-ns1/test-2")
+  (is (= 10 10))
+  (is (zero? veach))
+  (is (zero? vonce))
+  (is (zero? vcommon)))
+
+(deftest test-3
+  (println "Running: circleci.sample-test-ns1/test-3")
+  (is (= 109 109))
+  (is (zero? veach))
+  (is (zero? vonce))
+  (is (zero? vcommon)))
+
+
+(defn test-ns-hook
+  []
+  (test-1)
+  (test-2)
+  (test-3))
+
+
+(defn each-fixture
+  [f]
+  (binding [veach (inc veach)
+            vcommon (inc vcommon)]
+    (f)))
+
+
+(defn once-fixture
+  [f]
+  (binding [vonce (inc vonce)
+            vcommon (inc vcommon)]
+    (f)))
+
+
+
+(use-fixtures :each each-fixture)
+(use-fixtures :once once-fixture)

dev-resources/weasley/test/weasley/sample_test_ns2.clj

@@ -0,0 +1,65 @@
+(ns weasley.sample-test-ns2
+  (:require [clojure.test :refer :all]))
+
+(def ^:dynamic vonce 0)
+(def ^:dynamic veach 0)
+(def ^:dynamic vcommon 0)
+(def ^:dynamic level-of-nesting 0)
+
+(deftest test-1
+  (when (zero? level-of-nesting)
+    (println "Running: circleci.sample-test-ns2/test-1"))
+  (is (= 1 1))
+  (is (= 1 veach))
+  (is (= 1 vonce))
+  (is (= 2 vcommon)))
+
+(deftest test-2
+  (when (zero? level-of-nesting)
+    (println "Running: circleci.sample-test-ns2/test-2"))
+  (is (= 10 10))
+  (is (= 1 veach))
+  (is (= 1 vonce))
+  (is (= 2 vcommon)))
+
+(deftest test-3
+  (when (zero? level-of-nesting)
+    (println "Running: circleci.sample-test-ns2/test-3"))
+  (is (= 109 109))
+  (is (= 1 veach))
+  (is (= 1 vonce))
+  (is (= 2 vcommon)))
+
+(deftest ^:combination test-4
+  (when (zero? level-of-nesting)
+    (println "Running: circleci.sample-test-ns2/test-4"))
+  (binding [level-of-nesting (inc level-of-nesting)]
+    (test-1)
+    (test-2)
+    (test-3)))
+
+(deftest ^:combination test-5
+  (println "Running: circleci.sample-test-ns2/test-5")
+  (binding [level-of-nesting (inc level-of-nesting)]
+    (test-1)
+    (test-2)
+    (test-3)
+    (test-4)))
+
+
+(defn each-fixture
+  [f]
+  (binding [veach (inc veach)
+            vcommon (inc vcommon)]
+    (f)))
+
+
+(defn once-fixture
+  [f]
+  (binding [vonce (inc vonce)
+            vcommon (inc vcommon)]
+    (f)))
+
+
+(use-fixtures :each each-fixture)
+(use-fixtures :once once-fixture)

dev-resources/weasley/test/weasley/sample_test_ns3.clj

@@ -0,0 +1,73 @@
+(ns ^:global-fixture weasley.sample-test-ns3
+  (:require [clojure.test :refer :all]))
+
+(def ^:dynamic vonce 0)
+(def ^:dynamic veach 0)
+(def ^:dynamic vcommon 0)
+(def ^:dynamic level-of-nesting 0)
+
+(defn global-fixture-file
+  []
+  (slurp "global_fixture_test.out"))
+
+(deftest test-1
+  (when (zero? level-of-nesting)
+    (println "Running: circleci.sample-test-ns3/test-1"))
+  (is (= 1 veach))
+  (is (= 1 vonce))
+  (is (= 2 vcommon))
+  (is (= "1" (global-fixture-file))))
+
+(deftest test-2
+  (when (zero? level-of-nesting)
+    (println "Running: circleci.sample-test-ns3/test-2"))
+  (is (= 1 veach))
+  (is (= 1 vonce))
+  (is (= 2 vcommon))
+  (is (= "1" (global-fixture-file))))
+
+(deftest test-3
+  (when (zero? level-of-nesting)
+    (println "Running: circleci.sample-test-ns3/test-3"))
+  (is (= 1 veach))
+  (is (= 1 vonce))
+  (is (= 2 vcommon))
+  (is (= "1" (global-fixture-file))))
+
+(deftest ^:combination test-4
+  (when (zero? level-of-nesting)
+    (println "Running: circleci.sample-test-ns3/test-4"))
+  (binding [level-of-nesting (inc level-of-nesting)]
+    (test-1)
+    (test-2)
+    (test-3)
+    (assert (= "1" (global-fixture-file)))))
+
+(deftest ^:combination test-5
+  (println "Running: circleci.sample-test-ns3/test-5")
+  (binding [level-of-nesting (inc level-of-nesting)]
+    (test-1)
+    (test-2)
+    (test-3)
+    (test-4)
+    (assert (= "1" (global-fixture-file)))))
+
+
+(defn each-fixture
+  [f]
+  (binding [veach (inc veach)
+            vcommon (inc vcommon)]
+    (assert (= "1" (global-fixture-file)))
+    (f)))
+
+
+(defn once-fixture
+  [f]
+  (binding [vonce (inc vonce)
+            vcommon (inc vcommon)]
+    (assert (= "1" (global-fixture-file)))
+    (f)))
+
+
+(use-fixtures :each each-fixture)
+(use-fixtures :once once-fixture)

test.sh

@@ -0,0 +1,103 @@
+grep defproject project.clj | cut -d" " -f3 | xargs -I {} echo "(def circleci-test-version \"{}\")" > dev-resources/weasley/circleci.test.version
+
+lein install
+
+cd dev-resources/weasley/
+
+lein do clean, compile
+
+# Counts between leiningen and circleci are off because of tests with global fixtures (ns3)
+lein test > test_out.txt &&
+grep "Ran 33 tests containing 108 assertions." test_out.txt &&
+lein update-in :aliases empty -- update-in : assoc :test-selectors '{:default (complement :global-fixture)}'  -- test > test_out.txt &&
+grep "Ran 18 tests containing 60 assertions." test_out.txt &&
+
+lein tests weasley.sample-test-ns1 weasley.sample-test-ns2 > test_out.txt &&
+grep "Ran 18 tests containing 60 assertions." test_out.txt &&
+lein update-in :aliases empty -- test :only weasley.sample-test-ns1 weasley.sample-test-ns2 > test_out.txt &&
+grep "Ran 18 tests containing 60 assertions." test_out.txt &&
+
+lein tests weasley.sample-test-ns1 > test_out.txt &&
+grep "Ran 3 tests containing 12 assertions." test_out.txt &&
+lein update-in :aliases empty -- test :only weasley.sample-test-ns1 > test_out.txt &&
+grep "Ran 3 tests containing 12 assertions." test_out.txt &&
+
+lein tests weasley.sample-test-ns2 > test_out.txt &&
+grep "Ran 15 tests containing 48 assertions." test_out.txt &&
+lein update-in :aliases empty -- test :only weasley.sample-test-ns2 > test_out.txt &&
+grep "Ran 15 tests containing 48 assertions." test_out.txt &&
+
+lein tests weasley.sample-test-ns2 weasley.sample-test-ns2 weasley.sample-test-ns2 weasley.sample-test-ns2 > test_out.txt &&
+grep "Ran 15 tests containing 48 assertions." test_out.txt &&
+lein update-in :aliases empty -- test :only weasley.sample-test-ns2 weasley.sample-test-ns2 weasley.sample-test-ns2 > test_out.txt &&
+grep "Ran 15 tests containing 48 assertions." test_out.txt &&
+
+lein tests x y/z > test_out.txt &&
+grep "Ran 0 tests containing 0 assertions." test_out.txt &&
+lein update-in :aliases empty -- test :only x y/z > test_out.txt &&
+grep "Ran 0 tests containing 0 assertions." test_out.txt &&
+
+lein tests weasley.sample-test-ns2 weasley.sample-test-ns1/test-1 x y/z > test_out.txt &&
+grep "Ran 16 tests containing 52 assertions." test_out.txt &&
+lein update-in :aliases empty -- test :only weasley.sample-test-ns2 weasley.sample-test-ns1/test-1 x y/z > test_out.txt &&
+grep "Ran 16 tests containing 52 assertions." test_out.txt &&
+
+lein tests weasley.sample-test-ns2/test-1 > test_out.txt &&
+grep "Ran 1 tests containing 4 assertions." test_out.txt &&
+lein update-in :aliases empty -- test :only weasley.sample-test-ns2/test-1 > test_out.txt &&
+grep "Ran 1 tests containing 4 assertions." test_out.txt &&
+
+lein tests weasley.sample-test-ns2/test-1 weasley.sample-test-ns2/test-2 > test_out.txt &&
+grep "Ran 2 tests containing 8 assertions." test_out.txt &&
+lein update-in :aliases empty -- test :only weasley.sample-test-ns2/test-1 weasley.sample-test-ns2/test-2 > test_out.txt &&
+grep "Ran 2 tests containing 8 assertions." test_out.txt &&
+
+lein tests weasley.sample-test-ns2/test-1 weasley.sample-test-ns2 weasley.sample-test-ns2/test-2 weasley.sample-test-ns2/test-1 > test_out.txt &&
+grep "Ran 15 tests containing 48 assertions." test_out.txt &&
+lein update-in :aliases empty -- test :only weasley.sample-test-ns2/test-1 weasley.sample-test-ns2 weasley.sample-test-ns2/test-2 weasley.sample-test-ns2/test-1 > test_out.txt &&
+grep "Ran 15 tests containing 48 assertions." test_out.txt &&
+
+lein tests :select-vars weasley.sample-test-ns2 > test_out.txt &&
+grep "Ran 1 tests containing 4 assertions." test_out.txt &&
+lein update-in :aliases empty -- test :only weasley.sample-test-ns2/test-2 > test_out.txt &&
+grep "Ran 1 tests containing 4 assertions." test_out.txt &&
+
+lein tests :select-vars weasley.sample-test-ns1 > test_out.txt &&
+grep "Ran 1 tests containing 4 assertions." test_out.txt &&
+lein update-in :aliases empty -- test :only weasley.sample-test-ns1/test-2 > test_out.txt &&
+grep "Ran 1 tests containing 4 assertions." test_out.txt &&
+
+# The equivalent run for this in Leiningen does not work (test-ns-hook)
+lein tests :combination weasley.sample-test-ns2 > test_out.txt &&
+grep "Ran 12 tests containing 36 assertions." test_out.txt &&
+
+# The equivalent run for this in Leiningen does not work (test calling another test)
+lein tests :combination weasley.sample-test-ns2/test-4 weasley.sample-test-ns2/test-5 weasley.sample-test-ns2/test-1 > test_out.txt &&
+grep "Ran 12 tests containing 36 assertions." test_out.txt &&
+
+lein tests :combination weasley.sample-test-ns1 > test_out.txt &&
+grep "Ran 0 tests containing 0 assertions." test_out.txt &&
+lein update-in :aliases empty -- update-in : assoc :test-selectors '{:combination :combination}' -- test weasley.sample-test-ns1 :combination > test_out.txt &&
+grep "Ran 0 tests containing 0 assertions." test_out.txt &&
+
+# The equivalent run for this in Leiningen does not work (test calling another test)
+lein tests :combination weasley.sample-test-ns2/test-4 > test_out.txt &&
+grep "Ran 4 tests containing 12 assertions." test_out.txt &&
+
+# Global fixtures tests
+lein tests weasley.sample-test-ns3 > test_out.txt &&
+grep "Ran 15 tests containing 48 assertions." test_out.txt &&
+
+lein tests weasley.sample-test-ns3/test-1 > test_out.txt &&
+grep "Ran 1 tests containing 4 assertions." test_out.txt &&
+
+lein tests weasley.sample-test-ns3/test-1 weasley.sample-test-ns3/test-2 > test_out.txt &&
+grep "Ran 2 tests containing 8 assertions." test_out.txt &&
+
+lein tests :select-vars weasley.sample-test-ns3/test-1 weasley.sample-test-ns3/test-2 > test_out.txt &&
+grep "Ran 1 tests containing 4 assertions." test_out.txt &&
+
+lein tests :combination weasley.sample-test-ns3 > test_out.txt &&
+grep "Ran 12 tests containing 36 assertions." test_out.txt &&
+
+rm -f test_out.txt