Merge pull request #317 from magento-gl/4.3.3-Release-CheckList

4.3.3 release check list

CHANGELOG.md

@@ -1,6 +1,14 @@
 Magento Functional Testing Framework Changelog
 ================================================
 
+4.3.3
+---------
+### Enhancements
+* Enhance the details in the testgroupmembership.txt file.
+
+### Fixes
+* Fixed MFTF helpers & actionGroups allow duplicate argument names to be passed.
+
 4.3.2
 ---------
 ### Enhancements

composer.json

@@ -2,7 +2,7 @@
     "name": "magento/magento2-functional-testing-framework",
     "description": "Magento2 Functional Testing Framework",
     "type": "library",
-    "version": "4.3.2",
+    "version": "4.3.3",
     "license": "AGPL-3.0",
     "keywords": ["magento", "automation", "functional", "testing"],
     "config": {

dev/tests/unit/Magento/FunctionalTestFramework/Suite/SuiteGeneratorTest.php

@@ -40,6 +40,60 @@ protected function setUp(): void
         TestLoggingUtil::getInstance()->setMockLoggingUtil();
     }
 
+    /**
+     * Tests generating a suite given a set of parsed test data.
+     *
+     * @return void
+     * @throws Exception
+     */
+    public function testGenerateTestgroupmembership(): void
+    {
+        $suiteDataArrayBuilder = new SuiteDataArrayBuilder();
+        $mockSuiteData = $suiteDataArrayBuilder
+            ->withName('mockSuite')
+            ->includeGroups(['group1'])
+            ->build();
+        $testDataArrayBuilder = new TestDataArrayBuilder();
+        $mockSimpleTest1 = $testDataArrayBuilder
+            ->withName('simpleTest1')
+            ->withAnnotations(['group' => [['value' => 'group1']]])
+            ->withTestReference("NonExistantTest")
+            ->withTestActions()
+            ->build();
+        $mockSimpleTest2 = $testDataArrayBuilder
+            ->withName('simpleTest2')
+            ->withAnnotations(['group' => [['value' => 'group1']]])
+            ->withTestActions()
+            ->build();
+        $mockSimpleTest3 = $testDataArrayBuilder
+            ->withName('simpleTest3')
+            ->withAnnotations(['group' => [['value' => 'group1']]])
+            ->withTestActions()
+            ->build();
+        $mockTestData = array_merge($mockSimpleTest1, $mockSimpleTest2, $mockSimpleTest3);
+        $this->setMockTestAndSuiteParserOutput($mockTestData, $mockSuiteData);
+
+        // Make manifest for split suites
+        $suiteConfig = [
+            'mockSuite' => [
+                'mockSuite_0_G' => ['simpleTest1', 'simpleTest2'],
+                'mockSuite_1_G' => ['simpleTest3'],
+            ],
+        ];
+        $manifest = TestManifestFactory::makeManifest('default', $suiteConfig);
+
+        // parse and generate suite object with mocked data and manifest
+        $mockSuiteGenerator = SuiteGenerator::getInstance();
+        $mockSuiteGenerator->generateAllSuites($manifest);
+
+        // assert last split suite group generated
+        TestLoggingUtil::getInstance()->validateMockLogStatement(
+            'info',
+            'suite generated',
+            ['suite' => 'mockSuite_1_G', 'relative_path' => '_generated' . DIRECTORY_SEPARATOR . 'mockSuite_1_G']
+        );
+    }
+
     /**
      * Tests generating a single suite given a set of parsed test data.
      *

dev/tests/unit/Magento/FunctionalTestFramework/Util/TestGeneratorTest.php

@@ -24,6 +24,7 @@
 use ReflectionProperty;
 use tests\unit\Util\MagentoTestCase;
 use tests\unit\Util\TestLoggingUtil;
+use Magento\FunctionalTestingFramework\Exceptions\TestFrameworkException;
 
 class TestGeneratorTest extends MagentoTestCase
 {
@@ -299,6 +300,104 @@ function ($filename) use (&$generatedTests) {
         $this->assertArrayNotHasKey('test2Cest', $generatedTests);
     }
 
+    /**
+     * Test for exception thrown when duplicate arguments found
+     *
+     * @return void
+     * @throws TestFrameworkException
+     */
+    public function testIfExceptionThrownWhenDuplicateArgumentsFound()
+    {
+        $fileContents = '<actionGroups xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+            xsi:noNamespaceSchemaLocation="urn:magento:mftf:Test/etc/actionGroupSchema.xsd">
+              <actionGroup name="ActionGroupReturningValueActionGroup">
+                  <arguments>
+                      <argument name="count" type="string"/>
+                      <argument name="count" type="string"/>
+                  </arguments>
+                  <grabMultiple selector="selector" stepKey="grabProducts1"/>
+                  <assertCount stepKey="assertCount">
+                      <expectedResult type="int">{{count}}</expectedResult>
+                      <actualResult type="variable">grabProducts1</actualResult>
+                  </assertCount>
+                  <return value="{$grabProducts1}" stepKey="returnProducts1"/>
+              </actionGroup>
+          </actionGroups>';
+        $actionInput = 'fakeInput';
+        $actionObject = new ActionObject('fakeAction', 'comment', [
+          'userInput' => $actionInput
+        ]);
+        $annotation1 = ['group' => ['someGroupValue']];
+
+        $test1 = new TestObject(
+            'test1',
+            ['fakeAction' => $actionObject],
+            $annotation1,
+            [],
+            'filename'
+        );
+        $annotation2 = ['group' => ['someOtherGroupValue']];
+
+        $test2 = new TestObject(
+            'test2',
+            ['fakeAction' => $actionObject],
+            $annotation2,
+            [],
+            'filename'
+        );
+        $testGeneratorObject = TestGenerator::getInstance('', ['sampleTest' => $test1, 'test2' => $test2]);
+        $result = $testGeneratorObject->throwExceptionIfDuplicateArgumentsFound($testGeneratorObject);
+        $this->assertEquals($result, "");
+    }
+
+    /**
+     * Test for exception not thrown when duplicate arguments not found
+     *
+     * @return void
+     */
+    public function testIfExceptionNotThrownWhenDuplicateArgumentsNotFound()
+    {
+        $fileContents = '<actionGroups xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+              xsi:noNamespaceSchemaLocation="urn:magento:mftf:Test/etc/actionGroupSchema.xsd">
+                <actionGroup name="ActionGroupReturningValueActionGroup">
+                    <arguments>
+                        <argument name="count" type="string"/>
+                    </arguments>
+                    <grabMultiple selector="selector" stepKey="grabProducts1"/>
+                    <assertCount stepKey="assertCount">
+                        <expectedResult type="int">{{count}}</expectedResult>
+                        <actualResult type="variable">grabProducts1</actualResult>
+                    </assertCount>
+                    <return value="{$grabProducts1}" stepKey="returnProducts1"/>
+                </actionGroup>
+            </actionGroups>';
+        $actionInput = 'fakeInput';
+        $actionObject = new ActionObject('fakeAction', 'comment', [
+          'userInput' => $actionInput
+        ]);
+        $annotation1 = ['group' => ['someGroupValue']];
+
+        $test1 = new TestObject(
+            'test1',
+            ['fakeAction' => $actionObject],
+            $annotation1,
+            [],
+            'filename'
+        );
+        $annotation2 = ['group' => ['someOtherGroupValue']];
+
+        $test2 = new TestObject(
+            'test2',
+            ['fakeAction' => $actionObject],
+            $annotation2,
+            [],
+            'filename'
+        );
+        $testGeneratorObject = TestGenerator::getInstance('', ['sampleTest' => $test1, 'test2' => $test2]);
+        $result = $testGeneratorObject->throwExceptionIfDuplicateArgumentsFound($testGeneratorObject);
+        $this->assertEquals($result, "");
+    }
+
     /**
      * Tests that TestGenerator createAllTestFiles correctly filters based on group.
      *

src/Magento/FunctionalTestingFramework/Console/GenerateTestsCommand.php

@@ -231,9 +231,9 @@ protected function execute(InputInterface $input, OutputInterface $output)
                 $testManifest->createTestGroups($configNumber);
             }
 
-            SuiteGenerator::getInstance()->generateAllSuites($testManifest);
-
             $testManifest->generate();
+
+            SuiteGenerator::getInstance()->generateAllSuites($testManifest);
         } catch (\Exception $e) {
             if (!empty(GenerationErrorHandler::getInstance()->getAllErrors())) {
                 GenerationErrorHandler::getInstance()->printErrorSummary();

src/Magento/FunctionalTestingFramework/Suite/SuiteGenerator.php

@@ -95,6 +95,7 @@ public static function getInstance(): SuiteGenerator
      */
     public function generateAllSuites($testManifest)
     {
+        $this->generateTestgroupmembership($testManifest);
         $suites = $testManifest->getSuiteConfig();
 
         foreach ($suites as $suiteName => $suiteContent) {
@@ -139,6 +140,108 @@ public function generateSuite($suiteName)
         $this->generateSuiteFromTest($suiteName, []);
     }
 
+    /**
+     * Function which generate Testgroupmembership file.
+     *
+     * @param object $testManifest
+     * @return void
+     * @throws \Exception
+     */
+    public function generateTestgroupmembership($testManifest): void
+    {
+        $suites = $this->getSuitesDetails($testManifest);
+
+        // Path to groups folder
+        $baseDir = FilePathFormatter::format(TESTS_MODULE_PATH);
+        $path = $baseDir .'_generated/groups';
+
+        $allGroupsContent = $this->readAllGroupFiles($path);
+
+        // Output file path
+        $memberShipFilePath = $baseDir.'_generated/testgroupmembership.txt';
+        $testCaseNumber = 0;
+
+        if (!empty($allGroupsContent)) {
+            foreach ($allGroupsContent as $groupId => $groupInfo) {
+                foreach ($groupInfo as $testName) {
+                    // If file has -g then it is test suite
+                    if (str_contains($testName, '-g')) {
+                        $suitename = explode(" ", $testName);
+                        $suitename[1] = trim($suitename[1]);
+
+                        if (!empty($suites[$suitename[1]])) {
+                            foreach ($suites[$suitename[1]] as $key => $test) {
+                                $suiteTest = sprintf('%s:%s:%s:%s', $groupId, $key, $suitename[1], $test);
+                                file_put_contents($memberShipFilePath, $suiteTest . PHP_EOL, FILE_APPEND);
+                            }
+                        }
+                    } else {
+                        $defaultSuiteTest = sprintf('%s:%s:%s', $groupId, $testCaseNumber, $testName);
+                        file_put_contents($memberShipFilePath, $defaultSuiteTest, FILE_APPEND);
+                    }
+                    $testCaseNumber++;
+                }
+                $testCaseNumber = 0;
+            }
+        }
+    }
+
+    /**
+     * Function to format suites details
+     *
+     * @param object $testManifest
+     * @return array $suites
+     */
+    private function getSuitesDetails($testManifest): array
+    {
+        // Get suits and subsuites data array
+        $suites = $testManifest->getSuiteConfig();
+
+        // Add subsuites array[2nd dimension] to main array[1st dimension] to access it directly later
+        if (!empty($suites)) {
+            foreach ($suites as $subSuites) {
+                if (!empty($subSuites)) {
+                    foreach ($subSuites as $subSuiteName => $suiteTestNames) {
+                        if (!is_numeric($subSuiteName)) {
+                            $suites[$subSuiteName] = $suiteTestNames;
+                        } else {
+                            continue;
+                        }
+                    }
+                }
+            }
+        }
+        return $suites;
+    }
+
+  /**
+   * Function to read all group* text files inside /groups folder
+   *
+   * @param object $path
+   * @return array $allGroupsContent
+   */
+    private function readAllGroupFiles($path): array
+    {
+        // Read all group files
+        if (is_dir($path)) {
+            $groupFiles = glob("$path/group*.txt");
+            if ($groupFiles === false) {
+                throw new RuntimeException("glob(): error with '$path'");
+            }
+            sort($groupFiles, SORT_NATURAL);
+        }
+
+        // Read each file in the reverse order and form an array with groupId as key
+        $groupNumber = 0;
+        $allGroupsContent = [];
+        while (!empty($groupFiles)) {
+            $group = array_pop($groupFiles);
+            $allGroupsContent[$groupNumber] = file($group);
+            $groupNumber++;
+        }
+        return $allGroupsContent;
+    }
+
     /**
      * Function which takes a suite name and a set of test names. The function then generates all relevant supporting
      * files and classes for the suite. The function takes an optional argument for suites which are split by a parallel