Skip to content

Commit ff00b07

Browse files
GrzegorzDrozdDavertMik
GrzegorzDrozd
authored and
DavertMik
committed
Be aware of test dependency during split (#46)
* Be aware of test dependency during split * Test fixes New tests Test fixtures * Travis fixes * add verbose to see why tests fail * Build fix * still fails :/ * Test if correct code is executed. * Test if correct code is executed. * Test if correct code is executed. * Test if correct code is executed. * Test if correct code is executed. * Test if correct code is executed. * dev option * dev option * test what file is used * test what file is used * debug * debug * debug * debug * debug * temp disable some tests * test * test * test * test * test * test * test * remove debug * restore minimal version of codeception dependency * restore minimal version of codeception dependency with php 7.0 * fix test cases
1 parent 3f3b933 commit ff00b07

File tree

11 files changed

+2012
-580
lines changed

11 files changed

+2012
-580
lines changed

‎.travis.yml

+2-2
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,7 @@ php:
1010
- 7.1
1111

1212
before_script:
13-
- composer install
13+
- composer install --dev
1414

1515
script:
16-
- composer exec phpunit -- tests --bootstrap tests/bootstrap.php
16+
- composer -v exec phpunit -- tests --bootstrap tests/bootstrap.php --exclude-group example --stderr -v --debug

‎composer.json

+2-1
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@
1717
"consolidation/robo": "^1.0.0"
1818
},
1919
"require-dev": {
20-
"codeception/base": "~2.2"
20+
"codeception/base": "^2.2",
21+
"codeception/codeception": "^2.2"
2122
}
2223
}

‎composer.lock

+1,674-566
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎src/SplitTestsByGroups.php

+171-6
Original file line numberDiff line numberDiff line change
@@ -13,11 +13,21 @@
1313

1414
trait SplitTestsByGroups
1515
{
16+
/**
17+
* @param $numGroups
18+
*
19+
* @return SplitTestsByGroupsTask
20+
*/
1621
protected function taskSplitTestsByGroups($numGroups)
1722
{
1823
return $this->task(SplitTestsByGroupsTask::class, $numGroups);
1924
}
2025

26+
/**
27+
* @param $numGroups
28+
*
29+
* @return SplitTestFilesByGroupsTask
30+
*/
2131
protected function taskSplitTestFilesByGroups($numGroups)
2232
{
2333
return $this->task(SplitTestFilesByGroupsTask::class, $numGroups);
@@ -64,6 +74,77 @@ public function excludePath($path)
6474

6575
return $this;
6676
}
77+
78+
/**
79+
* @param $item
80+
* @param array $items
81+
* @param array $resolved
82+
* @param array $unresolved
83+
*
84+
* @return array
85+
*/
86+
protected function resolveDependencies($item, array $items, array $resolved, array $unresolved) {
87+
$unresolved[] = $item;
88+
foreach ($items[$item] as $dep) {
89+
if (!in_array($dep, $resolved)) {
90+
if (!in_array($dep, $unresolved)) {
91+
$unresolved[] = $dep;
92+
list($resolved, $unresolved) = $this->resolveDependencies($dep, $items, $resolved, $unresolved);
93+
} else {
94+
throw new \RuntimeException("Circular dependency: $item -> $dep");
95+
}
96+
}
97+
}
98+
// Add $item to $resolved if it's not already there
99+
if (!in_array($item, $resolved)) {
100+
$resolved[] = $item;
101+
}
102+
// Remove all occurrences of $item in $unresolved
103+
while (($index = array_search($item, $unresolved)) !== false) {
104+
unset($unresolved[$index]);
105+
}
106+
107+
return [$resolved, $unresolved];
108+
}
109+
110+
/**
111+
* Make sure that tests are in array are always with full path and name.
112+
*
113+
* @param array $testsListWithDependencies
114+
*
115+
* @return array
116+
*/
117+
protected function resolveDependenciesToFullNames(array $testsListWithDependencies){
118+
// make sure that dependencies are in array as full names
119+
foreach ($testsListWithDependencies as $testName => $test) {
120+
foreach ($test as $i => $dependency) {
121+
122+
// sometimes it is written as class::method.
123+
// for that reason we do trim in first case and replace from :: to one in second case
124+
125+
126+
// just test name, that means that class name is the same, just different method name
127+
if (strrpos($dependency, ':') === false) {
128+
$testsListWithDependencies[$testName][$i] = trim(substr($testName,0,strrpos($testName, ':')), ':') . ':' . $dependency;
129+
continue;
130+
}
131+
$dependency = str_replace('::', ':', $dependency);
132+
// className:testName, that means we need to find proper test.
133+
list($targetTestFileName, $targetTestMethodName) = explode(':', $dependency);
134+
135+
// look for proper test in list of all tests. Test could be in different directory so we need to compare
136+
// strings and if matched we just assign found test name
137+
foreach (array_keys($testsListWithDependencies) as $arrayKey) {
138+
if (strpos($arrayKey, $targetTestFileName . '.php:' . $targetTestMethodName) !== false) {
139+
$testsListWithDependencies[$testName][$i] = $arrayKey;
140+
continue 2;
141+
}
142+
}
143+
throw new \RuntimeException('Dependency target test '.$dependency.' not found. Please make sure test exists and you are using full test name');
144+
}
145+
}
146+
return $testsListWithDependencies;
147+
}
67148
}
68149

69150
/**
@@ -89,19 +170,103 @@ public function run()
89170
$testLoader->loadTests($this->testsFrom);
90171
$tests = $testLoader->getTests();
91172

92-
$i = 0;
93-
$groups = [];
94-
95173
$this->printTaskInfo('Processing ' . count($tests) . ' tests');
96-
// splitting tests by groups
174+
175+
$testsHaveAtLeastOneDependency = false;
176+
177+
// test preloading (and fetching dependencies) requires dummy DI service.
178+
$di = new \Codeception\Lib\Di();
179+
// gather test dependencies and deal with dataproviders
180+
$testsListWithDependencies = [];
97181
foreach ($tests as $test) {
98182
if ($test instanceof DataProvider || $test instanceof DataProviderTestSuite) {
99183
$test = current($test->tests());
100184
}
101-
$groups[($i % $this->numGroups) + 1][] = TestDescriptor::getTestFullName($test);
102-
$i++;
185+
186+
// load dependencies for cest type. Unit tests dependencies are loaded automatically
187+
if ($test instanceof \Codeception\Test\Cest) {
188+
$test->getMetadata()->setServices(['di'=>$di]);
189+
$test->preload();
190+
}
191+
192+
if (method_exists($test, 'getMetadata')) {
193+
$testsListWithDependencies[TestDescriptor::getTestFullName($test)] = $test->getMetadata()
194+
->getDependencies();
195+
if ($testsHaveAtLeastOneDependency === false and count($test->getMetadata()->getDependencies()) != 0) {
196+
$testsHaveAtLeastOneDependency = true;
197+
}
198+
199+
// little hack to get dependencies from phpunit test cases that are private.
200+
} elseif ($test instanceof \PHPUnit\Framework\TestCase) {
201+
$ref = new \ReflectionObject($test);
202+
do {
203+
try{
204+
$property = $ref->getProperty('dependencies');
205+
$property->setAccessible(true);
206+
$testsListWithDependencies[TestDescriptor::getTestFullName($test)] = $property->getValue($test);
207+
208+
if ($testsHaveAtLeastOneDependency === false and count($property->getValue($test)) != 0) {
209+
$testsHaveAtLeastOneDependency = true;
210+
}
211+
212+
} catch (\ReflectionException $e) {
213+
// go up on level on inheritance chain.
214+
}
215+
} while($ref = $ref->getParentClass());
216+
217+
} else {
218+
$testsListWithDependencies[TestDescriptor::getTestFullName($test)] = [];
219+
}
103220
}
221+
222+
if ($testsHaveAtLeastOneDependency) {
223+
$this->printTaskInfo('Resolving test dependencies');
224+
225+
// make sure that dependencies are in array as full names
226+
try {
227+
$testsListWithDependencies = $this->resolveDependenciesToFullNames($testsListWithDependencies);
228+
} catch (\Exception $e) {
229+
$this->printTaskError($e->getMessage());
230+
return false;
231+
}
232+
233+
// resolved and ordered list of dependencies
234+
$orderedListOfTests = [];
235+
// helper array
236+
$unresolved = [];
104237

238+
// Resolve dependencies for each test
239+
foreach (array_keys($testsListWithDependencies) as $test) {
240+
try {
241+
list ($orderedListOfTests, $unresolved) = $this->resolveDependencies($test, $testsListWithDependencies, $orderedListOfTests, $unresolved);
242+
} catch (\Exception $e) {
243+
$this->printTaskError($e->getMessage());
244+
return false;
245+
}
246+
}
247+
248+
// if we don't have any dependencies just use keys from original list.
249+
} else {
250+
$orderedListOfTests = array_keys($testsListWithDependencies);
251+
}
252+
253+
// for even split, calculate number of tests in each group
254+
$numberOfElementsInGroup = floor(count($orderedListOfTests) / $this->numGroups);
255+
256+
$i = 1;
257+
$groups = [];
258+
259+
// split tests into files.
260+
foreach ($orderedListOfTests as $test) {
261+
// move to the next group ONLY if number of tests is equal or greater desired number of tests in group
262+
// AND current test has no dependencies AKA: we are in different branch than previous test
263+
if (!empty($groups[$i]) AND count($groups[$i]) >= $numberOfElementsInGroup AND $i <= ($this->numGroups-1) AND empty($testsListWithDependencies[$test])) {
264+
$i++;
265+
}
266+
267+
$groups[$i][] = $test;
268+
}
269+
105270
// saving group files
106271
foreach ($groups as $i => $tests) {
107272
$filename = $this->saveTo . $i;

‎tests/MergeJUnitReportsTest.php

+1-1
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
<?php
22

3-
class MergeJUnitReportsTest extends \PHPUnit_Framework_TestCase
3+
class MergeJUnitReportsTest extends \PHPUnit\Framework\TestCase
44
{
55
use Codeception\Task\MergeReports;
66

‎tests/SplitTestsByGroupsTaskTest.php

+72-3
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
<?php
22

3-
class SplitTestsByGroupsTaskTest extends PHPUnit_Framework_TestCase
3+
class SplitTestsByGroupsTaskTest extends \PHPUnit\Framework\TestCase
44
{
55
use \Codeception\Task\SplitTestsByGroups;
66

@@ -15,6 +15,7 @@ public function testGroupsCanBeSplit()
1515
for ($i = 1; $i <= 10; $i++) {
1616
$this->assertFileExists("tests/result/group_$i");
1717
}
18+
$this->assertFileNotExists("tests/result/group_11");
1819
}
1920

2021
public function testSplitFilesByGroups()
@@ -31,11 +32,79 @@ public function testSplitFilesByGroups()
3132
}
3233
}
3334

35+
/**
36+
* Test Circular dependency protection
37+
*
38+
* @throws \Robo\Exception\TaskException
39+
*/
40+
public function testCircularDependencyDetectionAndHandling(){
41+
$task = new Codeception\Task\SplitTestsByGroupsTask(5);
42+
$output = new \Symfony\Component\Console\Output\BufferedOutput();
43+
$task->setLogger(new \Consolidation\Log\Logger($output));
44+
$task->testsFrom('tests/fixtures/DependencyResolutionExampleTests2')
45+
->projectRoot('vendor/codeception/base/')
46+
->groupsTo('tests/result/group_')
47+
->run();
48+
49+
$d = $output->fetch();
50+
51+
self::assertContains('Circular dependency:', $d);
52+
53+
// make sure that no files were generated.
54+
$this->assertEmpty(glob("tests/result/group_*"));
55+
}
56+
57+
/**
58+
* Test dependency resolving
59+
*
60+
* @throws \Robo\Exception\TaskException
61+
*/
62+
public function testDependencyResolving(){
63+
64+
$task = new Codeception\Task\SplitTestsByGroupsTask(2);
65+
$output = new \Symfony\Component\Console\Output\BufferedOutput();
66+
$task->setLogger(new \Consolidation\Log\Logger($output));
67+
68+
$task->testsFrom('tests/fixtures/DependencyResolutionExampleTests')
69+
->projectRoot('vendor/codeception/base/')
70+
->groupsTo('tests/result/group_')
71+
->run();
72+
for ($i = 1; $i <= 2; $i++) {
73+
$this->assertFileExists("tests/result/group_$i");
74+
}
75+
76+
// because path might be different on every system we need only last part of the path.
77+
$firstFile = file_get_contents("tests/result/group_1");
78+
$lines = [];
79+
foreach(explode("\n", $firstFile) as $line) {
80+
$lines[] = substr($line, -22);
81+
}
82+
// correct order of test execution is preserved
83+
self::assertSame(['Example1Test.php:testB', 'Example1Test.php:testA', 'Example1Test.php:testC'], $lines);
84+
85+
// check second file.
86+
$secondFile = file_get_contents("tests/result/group_2");
87+
$lines = [];
88+
foreach(explode("\n", $secondFile) as $line) {
89+
$lines[] = substr($line, -22);
90+
}
91+
// correct order of test execution is preserved
92+
self::assertSame(['Example2Test.php:testE', 'Example2Test.php:testD', 'Example3Test.php:testF', 'Example3Test.php:testG'], $lines);
93+
}
94+
3495
public function setUp()
3596
{
3697
@mkdir('tests/result');
37-
for ($i = 1; $i <= 10; $i++) {
38-
@unlink("tests/result/group_$i");
98+
99+
// remove all files even from bad runs.
100+
foreach(glob('tests/result/group_*') as $file) {
101+
$file = new SplFileInfo($file);
102+
if (is_file($file)) {
103+
@unlink($file);
104+
}
39105
}
40106
}
107+
108+
109+
41110
}

‎tests/bootstrap.php

+4-1
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,9 @@
11
<?php
2+
//require_once 'vendor/autoload.php';
3+
require_once 'vendor/codeception/codeception/autoload.php';
4+
25
\Robo\Robo::createDefaultContainer(
36
new Symfony\Component\Console\Input\ArrayInput([]),
47
new Symfony\Component\Console\Output\NullOutput()
58
);
6-
//\Robo\Robo::getContainer()->add('logger', new \Consolidation\Log\Logger(new \Symfony\Component\Console\Output\NullOutput));
9+
//\Robo\Robo::getContainer()->add('logger', new \Consolidation\Log\Logger(new \Symfony\Component\Console\Output\NullOutput));

‎tests/fixtures/DependencyResolutionExampleTests/DirA/Example1Test.php

+28
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,28 @@
1+
<?php
2+
3+
class Example1Test extends \PHPUnit\Framework\TestCase {
4+
5+
/**
6+
* @depends testB
7+
* @group example
8+
*/
9+
public function testA() {
10+
self::assertTrue(true);
11+
}
12+
13+
14+
/**
15+
* @group example
16+
*/
17+
public function testB(){
18+
self::assertTrue(true);
19+
}
20+
21+
/**
22+
* @depends testA
23+
* @group example
24+
*/
25+
public function testC(){
26+
self::assertTrue(true);
27+
}
28+
}

‎tests/fixtures/DependencyResolutionExampleTests/DirA/Example2Test.php

+20
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,20 @@
1+
<?php
2+
3+
class Example2Test extends \PHPUnit\Framework\TestCase{
4+
5+
6+
/**
7+
* @depends Example2Test::testE
8+
* @group example
9+
*/
10+
public function testD(){
11+
self::assertTrue(true);
12+
}
13+
14+
/**
15+
* @group example
16+
*/
17+
public function testE(){
18+
self::assertTrue(true);
19+
}
20+
}

0 commit comments

Comments
 (0)