[SUREFIRE-756] Allow ability to capture executed random runOrder for re-play purposes

winglam · winglam · commit 009bbbbba969 · 2020-08-25T23:30:20.000Z
diff --git a/maven-failsafe-plugin/src/main/java/org/apache/maven/plugin/failsafe/IntegrationTestMojo.java b/maven-failsafe-plugin/src/main/java/org/apache/maven/plugin/failsafe/IntegrationTestMojo.java
@@ -310,6 +310,22 @@ public class IntegrationTestMojo
     @Parameter( property = "failsafe.runOrder", defaultValue = "filesystem" )
     private String runOrder;
 
+    /**
+     * Sets the random seed that will be used to order the tests if {@code failsafe.runOrder} is set to {@code random}.
+     * <br>
+     * <br>
+     * If no seeds are set and {@code failsafe.runOrder} is set to {@code random}, then the seed used will be
+     * outputted (search for "To reproduce ordering use flag -Dfailsafe.runOrder.random.seed").
+     * <br>
+     * <br>
+     * To deterministically reproduce any random test order that was run before, simply set the seed to 
+     * be the same value.
+     *
+     * @since 3.0.0-M6
+     */
+    @Parameter( property = "failsafe.runOrder.random.seed" )
+    private Long runOrderRandomSeed;
+
     /**
      * A file containing include patterns, each in a next line. Blank lines, or lines starting with # are ignored.
      * If {@code includes} are also specified, these patterns are appended. Example with path, simple and regex
@@ -891,6 +907,18 @@ public void setRunOrder( String runOrder )
         this.runOrder = runOrder;
     }
 
+    @Override
+    public Long getRunOrderRandomSeed()
+    {
+        return runOrderRandomSeed;
+    }
+
+    @Override
+    public void setRunOrderRandomSeed( Long runOrderRandomSeed )
+    {
+        this.runOrderRandomSeed = runOrderRandomSeed;
+    }
+
     @Override
     public File getIncludesFile()
     {
diff --git a/maven-surefire-common/src/main/java/org/apache/maven/plugin/surefire/AbstractSurefireMojo.java b/maven-surefire-common/src/main/java/org/apache/maven/plugin/surefire/AbstractSurefireMojo.java
@@ -865,6 +865,10 @@ public abstract class AbstractSurefireMojo
 
     public abstract void setRunOrder( String runOrder );
 
+    public abstract Long getRunOrderRandomSeed();
+
+    public abstract void setRunOrderRandomSeed( Long runOrderRandomSeed );
+
     protected abstract void handleSummary( RunResult summary, Exception firstForkException )
         throws MojoExecutionException, MojoFailureException;
 
@@ -1129,6 +1133,7 @@ boolean verifyParameters()
             warnIfNotApplicableSkipAfterFailureCount();
             warnIfIllegalTempDir();
             warnIfForkCountIsZero();
+            printDefaultSeedIfNecessary();
         }
         return true;
     }
@@ -1285,7 +1290,7 @@ private RunResult executeProvider( @Nonnull ProviderInfo provider, @Nonnull Defa
         ClassLoaderConfiguration classLoaderConfiguration = getClassLoaderConfiguration();
         provider.addProviderProperties();
         RunOrderParameters runOrderParameters =
-            new RunOrderParameters( getRunOrder(), getStatisticsFile( getConfigChecksum() ) );
+            new RunOrderParameters( getRunOrder(), getStatisticsFile( getConfigChecksum() ), getRunOrderRandomSeed() );
 
         if ( isNotForking() )
         {
@@ -3051,6 +3056,17 @@ private void warnIfIllegalTempDir() throws MojoFailureException
         }
     }
 
+    private void printDefaultSeedIfNecessary()
+    {
+        if ( getRunOrderRandomSeed() == null && getRunOrder().equals( RunOrder.RANDOM.name() ) )
+        {
+            setRunOrderRandomSeed( System.nanoTime() );
+            getConsoleLogger().info(
+                "Tests will run in random order. To reproduce ordering use flag -D"
+                    + getPluginName() + ".runOrder.random.seed=" + getRunOrderRandomSeed() );
+        }
+    }
+
     final class TestNgProviderInfo
         implements ProviderInfo
     {
diff --git a/maven-surefire-common/src/main/java/org/apache/maven/plugin/surefire/booterclient/BooterSerializer.java b/maven-surefire-common/src/main/java/org/apache/maven/plugin/surefire/booterclient/BooterSerializer.java
@@ -56,6 +56,7 @@
 import static org.apache.maven.surefire.booter.BooterConstants.PLUGIN_PID;
 import static org.apache.maven.surefire.booter.BooterConstants.PROCESS_CHECKER;
 import static org.apache.maven.surefire.booter.BooterConstants.PROVIDER_CONFIGURATION;
+import static org.apache.maven.surefire.booter.BooterConstants.RUN_ORDER_RANDOM_SEED;
 import static org.apache.maven.surefire.booter.BooterConstants.REPORTSDIRECTORY;
 import static org.apache.maven.surefire.booter.BooterConstants.REQUESTEDTEST;
 import static org.apache.maven.surefire.booter.BooterConstants.RERUN_FAILING_TESTS_COUNT;
@@ -161,6 +162,7 @@ File serialize( KeyValueSource sourceProperties, ProviderConfiguration providerC
         {
             properties.setProperty( RUN_ORDER, RunOrder.asString( runOrderParameters.getRunOrder() ) );
             properties.setProperty( RUN_STATISTICS_FILE, runOrderParameters.getRunStatisticsFile() );
+            properties.setProperty( RUN_ORDER_RANDOM_SEED, runOrderParameters.getRunOrderRandomSeed() );
         }
 
         ReporterConfiguration reporterConfiguration = providerConfiguration.getReporterConfiguration();
diff --git a/maven-surefire-common/src/test/java/org/apache/maven/plugin/surefire/AbstractSurefireMojoJava7PlusTest.java b/maven-surefire-common/src/test/java/org/apache/maven/plugin/surefire/AbstractSurefireMojoJava7PlusTest.java
@@ -839,6 +839,18 @@ public void setRunOrder( String runOrder )
 
         }
 
+        @Override
+        public Long getRunOrderRandomSeed()
+        {
+            return null;
+        }
+
+        @Override
+        public void setRunOrderRandomSeed( Long runOrderRandomSeed )
+        {
+
+        }
+
         @Override
         protected void handleSummary( RunResult summary, Exception firstForkException )
         {
diff --git a/maven-surefire-common/src/test/java/org/apache/maven/plugin/surefire/AbstractSurefireMojoTest.java b/maven-surefire-common/src/test/java/org/apache/maven/plugin/surefire/AbstractSurefireMojoTest.java
@@ -2381,6 +2381,18 @@ public void setRunOrder( String runOrder )
 
         }
 
+        @Override
+        public Long getRunOrderRandomSeed()
+        {
+            return null;
+        }
+
+        @Override
+        public void setRunOrderRandomSeed( Long runOrderRandomSeed )
+        {
+
+        }
+
         @Override
         protected void handleSummary( RunResult summary, Exception firstForkException )
         {
diff --git a/maven-surefire-common/src/test/java/org/apache/maven/plugin/surefire/MojoMocklessTest.java b/maven-surefire-common/src/test/java/org/apache/maven/plugin/surefire/MojoMocklessTest.java
@@ -709,6 +709,18 @@ public void setRunOrder( String runOrder )
 
         }
 
+        @Override
+        public Long getRunOrderRandomSeed()
+        {
+            return null;
+        }
+
+        @Override
+        public void setRunOrderRandomSeed( Long runOrderRandomSeed )
+        {
+
+        }
+
         @Override
         public String[] getDependenciesToScan()
         {
diff --git a/maven-surefire-plugin/src/main/java/org/apache/maven/plugin/surefire/SurefirePlugin.java b/maven-surefire-plugin/src/main/java/org/apache/maven/plugin/surefire/SurefirePlugin.java
@@ -292,6 +292,22 @@ public class SurefirePlugin
     @Parameter( property = "surefire.runOrder", defaultValue = "filesystem" )
     private String runOrder;
 
+    /**
+     * Sets the random seed that will be used to order the tests if {@code surefire.runOrder} is set to {@code random}.
+     * <br>
+     * <br>
+     * If no seeds are set and {@code surefire.runOrder} is set to {@code random}, then the seed used will be
+     * outputted (search for "To reproduce ordering use flag -Dsurefire.runOrder.random.seed").
+     * <br>
+     * <br>
+     * To deterministically reproduce any random test order that was run before, simply set the seed to 
+     * be the same value.
+     *
+     * @since 3.0.0-M6
+     */
+    @Parameter( property = "surefire.runOrder.random.seed" )
+    private Long runOrderRandomSeed;
+
     /**
      * A file containing include patterns. Blank lines, or lines starting with # are ignored. If {@code includes} are
      * also specified, these patterns are appended. Example with path, simple and regex includes:
@@ -794,6 +810,18 @@ public void setRunOrder( String runOrder )
         this.runOrder = runOrder;
     }
 
+    @Override
+    public Long getRunOrderRandomSeed()
+    {
+        return runOrderRandomSeed;
+    }
+
+    @Override
+    public void setRunOrderRandomSeed( Long runOrderRandomSeed )
+    {
+        this.runOrderRandomSeed = runOrderRandomSeed;
+    }
+
     @Override
     public File getIncludesFile()
     {
diff --git a/surefire-api/src/main/java/org/apache/maven/surefire/api/testset/RunOrderParameters.java b/surefire-api/src/main/java/org/apache/maven/surefire/api/testset/RunOrderParameters.java
@@ -31,16 +31,34 @@ public class RunOrderParameters
 
     private File runStatisticsFile;
 
+    private Long runOrderRandomSeed;
+
     public RunOrderParameters( RunOrder[] runOrder, File runStatisticsFile )
     {
         this.runOrder = runOrder;
         this.runStatisticsFile = runStatisticsFile;
+        this.runOrderRandomSeed = null;
     }
 
     public RunOrderParameters( String runOrder, File runStatisticsFile )
     {
         this.runOrder = runOrder == null ? RunOrder.DEFAULT : RunOrder.valueOfMulti( runOrder );
         this.runStatisticsFile = runStatisticsFile;
+        this.runOrderRandomSeed = null;
+    }
+
+    public RunOrderParameters( RunOrder[] runOrder, File runStatisticsFile, Long runOrderRandomSeed )
+    {
+        this.runOrder = runOrder;
+        this.runStatisticsFile = runStatisticsFile;
+        this.runOrderRandomSeed = runOrderRandomSeed;
+    }
+
+    public RunOrderParameters( String runOrder, File runStatisticsFile, Long runOrderRandomSeed )
+    {
+        this.runOrder = runOrder == null ? RunOrder.DEFAULT : RunOrder.valueOfMulti( runOrder );
+        this.runStatisticsFile = runStatisticsFile;
+        this.runOrderRandomSeed = runOrderRandomSeed;
     }
 
     public static RunOrderParameters alphabetical()
@@ -53,6 +71,16 @@ public RunOrder[] getRunOrder()
         return runOrder;
     }
 
+    public Long getRunOrderRandomSeed()
+    {
+        return runOrderRandomSeed;
+    }
+
+    public void setRunOrderRandomSeed( Long runOrderRandomSeed )
+    {
+        this.runOrderRandomSeed = runOrderRandomSeed;
+    }
+
     public File getRunStatisticsFile()
     {
         return runStatisticsFile;
diff --git a/surefire-api/src/main/java/org/apache/maven/surefire/api/util/DefaultRunOrderCalculator.java b/surefire-api/src/main/java/org/apache/maven/surefire/api/util/DefaultRunOrderCalculator.java
@@ -28,6 +28,7 @@
 import java.util.Comparator;
 import java.util.LinkedHashSet;
 import java.util.List;
+import java.util.Random;
 
 /**
  * Applies the final runorder of the tests
@@ -45,12 +46,21 @@ public class DefaultRunOrderCalculator
 
     private final int threadCount;
 
+    private final Random random;
+
     public DefaultRunOrderCalculator( RunOrderParameters runOrderParameters, int threadCount )
     {
         this.runOrderParameters = runOrderParameters;
         this.threadCount = threadCount;
         this.runOrder = runOrderParameters.getRunOrder();
         this.sortOrder = this.runOrder.length > 0 ? getSortOrderComparator( this.runOrder[0] ) : null;
+        Long runOrderRandomSeed = runOrderParameters.getRunOrderRandomSeed();
+        if ( runOrderRandomSeed == null )
+        {
+            runOrderRandomSeed = System.nanoTime();
+            runOrderParameters.setRunOrderRandomSeed( runOrderRandomSeed );
+        }
+        this.random = new Random( runOrderRandomSeed );
     }
 
     @Override
@@ -72,7 +82,7 @@ private void orderTestClasses( List<Class<?>> testClasses, RunOrder runOrder )
     {
         if ( RunOrder.RANDOM.equals( runOrder ) )
         {
-            Collections.shuffle( testClasses );
+            Collections.shuffle( testClasses, random );
         }
         else if ( RunOrder.FAILEDFIRST.equals( runOrder ) )
         {
diff --git a/surefire-booter/src/main/java/org/apache/maven/surefire/booter/BooterConstants.java b/surefire-booter/src/main/java/org/apache/maven/surefire/booter/BooterConstants.java
@@ -46,6 +46,7 @@ private BooterConstants()
     public static final String SOURCE_DIRECTORY = "testSuiteDefinitionTestSourceDirectory";
     public static final String TEST_CLASSES_DIRECTORY = "testClassesDirectory";
     public static final String RUN_ORDER = "runOrder";
+    public static final String RUN_ORDER_RANDOM_SEED = "runOrderRandomSeed";
     public static final String RUN_STATISTICS_FILE = "runStatisticsFile";
     public static final String TEST_SUITE_XML_FILES = "testSuiteXmlFiles";
     public static final String PROVIDER_CONFIGURATION = "providerConfiguration";
diff --git a/surefire-booter/src/main/java/org/apache/maven/surefire/booter/BooterDeserializer.java b/surefire-booter/src/main/java/org/apache/maven/surefire/booter/BooterDeserializer.java
@@ -102,6 +102,7 @@ public ProviderConfiguration deserialize()
         final List<String> testSuiteXmlFiles = properties.getStringList( TEST_SUITE_XML_FILES );
         final File testClassesDirectory = properties.getFileProperty( TEST_CLASSES_DIRECTORY );
         final String runOrder = properties.getProperty( RUN_ORDER );
+        final Long runOrderRandomSeed = properties.getLongProperty( RUN_ORDER_RANDOM_SEED );
         final String runStatisticsFile = properties.getProperty( RUN_STATISTICS_FILE );
 
         final int rerunFailingTestsCount = properties.getIntProperty( RERUN_FAILING_TESTS_COUNT );
@@ -111,7 +112,8 @@ public ProviderConfiguration deserialize()
                                             properties.getBooleanProperty( FAILIFNOTESTS ), runOrder );
 
         RunOrderParameters runOrderParameters
-                = new RunOrderParameters( runOrder, runStatisticsFile == null ? null : new File( runStatisticsFile ) );
+                = new RunOrderParameters( runOrder, runStatisticsFile == null ? null : new File( runStatisticsFile ),
+                                          runOrderRandomSeed );
 
         TestArtifactInfo testNg = new TestArtifactInfo( testNgVersion, testArtifactClassifier );
         TestRequest testSuiteDefinition =
diff --git a/surefire-booter/src/main/java/org/apache/maven/surefire/booter/SurefireReflector.java b/surefire-booter/src/main/java/org/apache/maven/surefire/booter/SurefireReflector.java
@@ -178,10 +178,11 @@ private Object createRunOrderParameters( RunOrderParameters runOrderParameters )
             return null;
         }
         //Can't use the constructor with the RunOrder parameter. Using it causes some integration tests to fail.
-        Class<?>[] arguments = { String.class, File.class };
+        Class<?>[] arguments = { String.class, File.class, Long.class };
         Constructor<?> constructor = getConstructor( this.runOrderParameters, arguments );
         File runStatisticsFile = runOrderParameters.getRunStatisticsFile();
-        return newInstance( constructor, RunOrder.asString( runOrderParameters.getRunOrder() ), runStatisticsFile );
+        return newInstance( constructor, RunOrder.asString( runOrderParameters.getRunOrder() ), runStatisticsFile,
+                            runOrderParameters.getRunOrderRandomSeed() );
     }
 
     private Object createTestArtifactInfo( TestArtifactInfo testArtifactInfo )
diff --git a/surefire-its/src/test/java/org/apache/maven/surefire/its/RunOrderIT.java b/surefire-its/src/test/java/org/apache/maven/surefire/its/RunOrderIT.java
diff --git a/surefire-its/src/test/java/org/apache/maven/surefire/its/fixture/OutputValidator.java b/surefire-its/src/test/java/org/apache/maven/surefire/its/fixture/OutputValidator.java
diff --git a/surefire-its/src/test/java/org/apache/maven/surefire/its/fixture/SurefireLauncher.java b/surefire-its/src/test/java/org/apache/maven/surefire/its/fixture/SurefireLauncher.java

Original file line number	Diff line number	Diff line change
`@@ -56,6 +56,7 @@`
`56`	`56`	`import static org.apache.maven.surefire.booter.BooterConstants.PLUGIN_PID;`
`57`	`57`	`import static org.apache.maven.surefire.booter.BooterConstants.PROCESS_CHECKER;`
`58`	`58`	`import static org.apache.maven.surefire.booter.BooterConstants.PROVIDER_CONFIGURATION;`
	`59`	`+import static org.apache.maven.surefire.booter.BooterConstants.RUN_ORDER_RANDOM_SEED;`
`59`	`60`	`import static org.apache.maven.surefire.booter.BooterConstants.REPORTSDIRECTORY;`
`60`	`61`	`import static org.apache.maven.surefire.booter.BooterConstants.REQUESTEDTEST;`
`61`	`62`	`import static org.apache.maven.surefire.booter.BooterConstants.RERUN_FAILING_TESTS_COUNT;`
`@@ -161,6 +162,7 @@ File serialize( KeyValueSource sourceProperties, ProviderConfiguration providerC`
`161`	`162`	`{`
`162`	`163`	`properties.setProperty( RUN_ORDER, RunOrder.asString( runOrderParameters.getRunOrder() ) );`
`163`	`164`	`properties.setProperty( RUN_STATISTICS_FILE, runOrderParameters.getRunStatisticsFile() );`
	`165`	`+ properties.setProperty( RUN_ORDER_RANDOM_SEED, runOrderParameters.getRunOrderRandomSeed() );`
`164`	`166`	`}`
`165`	`167`
`166`	`168`	`ReporterConfiguration reporterConfiguration = providerConfiguration.getReporterConfiguration();`
Original file line number	Diff line number	Diff line change
`@@ -28,6 +28,7 @@`
`28`	`28`	`import java.util.Comparator;`
`29`	`29`	`import java.util.LinkedHashSet;`
`30`	`30`	`import java.util.List;`
	`31`	`+import java.util.Random;`
`31`	`32`
`32`	`33`	`/**`
`33`	`34`	`* Applies the final runorder of the tests`
`@@ -45,12 +46,21 @@ public class DefaultRunOrderCalculator`
`45`	`46`
`46`	`47`	`private final int threadCount;`
`47`	`48`
	`49`	`+ private final Random random;`
	`50`	`+`
`48`	`51`	`public DefaultRunOrderCalculator( RunOrderParameters runOrderParameters, int threadCount )`
`49`	`52`	`{`
`50`	`53`	`this.runOrderParameters = runOrderParameters;`
`51`	`54`	`this.threadCount = threadCount;`
`52`	`55`	`this.runOrder = runOrderParameters.getRunOrder();`
`53`	`56`	`this.sortOrder = this.runOrder.length > 0 ? getSortOrderComparator( this.runOrder[0] ) : null;`
	`57`	`+ Long runOrderRandomSeed = runOrderParameters.getRunOrderRandomSeed();`
	`58`	`+ if ( runOrderRandomSeed == null )`
	`59`	`+ {`
	`60`	`+ runOrderRandomSeed = System.nanoTime();`
	`61`	`+ runOrderParameters.setRunOrderRandomSeed( runOrderRandomSeed );`
	`62`	`+ }`
	`63`	`+ this.random = new Random( runOrderRandomSeed );`
`54`	`64`	`}`
`55`	`65`
`56`	`66`	`@Override`
`@@ -72,7 +82,7 @@ private void orderTestClasses( List<Class<?>> testClasses, RunOrder runOrder )`
`72`	`82`	`{`
`73`	`83`	`if ( RunOrder.RANDOM.equals( runOrder ) )`
`74`	`84`	`{`
`75`		`- Collections.shuffle( testClasses );`
	`85`	`+ Collections.shuffle( testClasses, random );`
`76`	`86`	`}`
`77`	`87`	`else if ( RunOrder.FAILEDFIRST.equals( runOrder ) )`
`78`	`88`	`{`