diff --git a/maven-failsafe-plugin/src/main/java/org/apache/maven/plugin/failsafe/IntegrationTestMojo.java b/maven-failsafe-plugin/src/main/java/org/apache/maven/plugin/failsafe/IntegrationTestMojo.java index 2a207c8aca..57674b080c 100644 --- a/maven-failsafe-plugin/src/main/java/org/apache/maven/plugin/failsafe/IntegrationTestMojo.java +++ b/maven-failsafe-plugin/src/main/java/org/apache/maven/plugin/failsafe/IntegrationTestMojo.java @@ -310,6 +310,10 @@ public class IntegrationTestMojo @Parameter( property = "failsafe.runOrder", defaultValue = "filesystem" ) private String runOrder; + //TODO docs + @Parameter( property = "failsafe.methodRunOrder", defaultValue = "default" ) + private String methodRunOrder; + /** * Sets the random seed that will be used to order the tests if {@code failsafe.runOrder} is set to {@code random}. *
@@ -907,6 +911,12 @@ public void setRunOrder( String runOrder ) this.runOrder = runOrder; } + @Override + public String getMethodRunOrder() + { + return methodRunOrder; + } + @Override public Long getRunOrderRandomSeed() { diff --git a/maven-surefire-common/src/main/java/org/apache/maven/plugin/surefire/AbstractSurefireMojo.java b/maven-surefire-common/src/main/java/org/apache/maven/plugin/surefire/AbstractSurefireMojo.java index 4d1af3dbef..893b92a90b 100644 --- a/maven-surefire-common/src/main/java/org/apache/maven/plugin/surefire/AbstractSurefireMojo.java +++ b/maven-surefire-common/src/main/java/org/apache/maven/plugin/surefire/AbstractSurefireMojo.java @@ -86,6 +86,7 @@ import org.apache.maven.surefire.api.testset.TestSetFailedException; import org.apache.maven.surefire.api.util.DefaultScanResult; import org.apache.maven.surefire.api.util.RunOrder; +import org.apache.maven.surefire.util.MethodRunOrder; import org.apache.maven.toolchain.DefaultToolchain; import org.apache.maven.toolchain.Toolchain; import org.apache.maven.toolchain.ToolchainManager; @@ -679,7 +680,7 @@ public abstract class AbstractSurefireMojo * * @since 2.2 */ - @Parameter( property = "trimStackTrace", defaultValue = "true" ) + @Parameter( property = "trimStackTrace", defaultValue = "false" ) private boolean trimStackTrace; /** @@ -865,6 +866,8 @@ public abstract class AbstractSurefireMojo public abstract void setRunOrder( String runOrder ); + public abstract String getMethodRunOrder(); + public abstract Long getRunOrderRandomSeed(); public abstract void setRunOrderRandomSeed( Long runOrderRandomSeed ); @@ -1291,7 +1294,8 @@ private RunResult executeProvider( @Nonnull ProviderInfo provider, @Nonnull Defa ClassLoaderConfiguration classLoaderConfiguration = getClassLoaderConfiguration(); provider.addProviderProperties(); RunOrderParameters runOrderParameters = - new RunOrderParameters( getRunOrder(), getStatisticsFile( getConfigChecksum() ), getRunOrderRandomSeed() ); + new RunOrderParameters( getRunOrder(), getStatisticsFile( getConfigChecksum() ), getRunOrderRandomSeed(), + getMethodRunOrder() ); if ( isNotForking() ) { @@ -2754,6 +2758,7 @@ private String getConfigChecksum() checksum.add( getObjectFactory() ); checksum.add( getFailIfNoTests() ); checksum.add( getRunOrder() ); + checksum.add( getMethodRunOrder().toString() ); checksum.add( getDependenciesToScan() ); checksum.add( getForkedProcessExitTimeoutInSeconds() ); checksum.add( getRerunFailingTestsCount() ); @@ -3068,7 +3073,9 @@ protected void warnIfIllegalFailOnFlakeCount() throws MojoFailureException private void printDefaultSeedIfNecessary() { - if ( getRunOrderRandomSeed() == null && getRunOrder().equals( RunOrder.RANDOM.name() ) ) + if ( getRunOrderRandomSeed() == null + && ( getRunOrder().equals( RunOrder.RANDOM.name() ) + || getMethodRunOrder().equals( MethodRunOrder.RANDOM.name() ) ) ) { setRunOrderRandomSeed( System.nanoTime() ); getConsoleLogger().info( diff --git a/maven-surefire-common/src/main/java/org/apache/maven/plugin/surefire/booterclient/BooterSerializer.java b/maven-surefire-common/src/main/java/org/apache/maven/plugin/surefire/booterclient/BooterSerializer.java index 1335a82ddf..174b4bd59e 100644 --- a/maven-surefire-common/src/main/java/org/apache/maven/plugin/surefire/booterclient/BooterSerializer.java +++ b/maven-surefire-common/src/main/java/org/apache/maven/plugin/surefire/booterclient/BooterSerializer.java @@ -54,6 +54,7 @@ import static org.apache.maven.surefire.booter.BooterConstants.INCLUDES_PROPERTY_PREFIX; import static org.apache.maven.surefire.booter.BooterConstants.ISTRIMSTACKTRACE; import static org.apache.maven.surefire.booter.BooterConstants.MAIN_CLI_OPTIONS; +import static org.apache.maven.surefire.booter.BooterConstants.METHOD_RUN_ORDER; import static org.apache.maven.surefire.booter.BooterConstants.PLUGIN_PID; import static org.apache.maven.surefire.booter.BooterConstants.PROCESS_CHECKER; import static org.apache.maven.surefire.booter.BooterConstants.PROVIDER_CONFIGURATION; @@ -163,6 +164,7 @@ File serialize( KeyValueSource sourceProperties, ProviderConfiguration providerC { properties.setProperty( RUN_ORDER, RunOrder.asString( runOrderParameters.getRunOrder() ) ); properties.setProperty( RUN_STATISTICS_FILE, runOrderParameters.getRunStatisticsFile() ); + properties.setProperty( METHOD_RUN_ORDER, runOrderParameters.getMethodRunOrder().toString() ); properties.setProperty( RUN_ORDER_RANDOM_SEED, runOrderParameters.getRunOrderRandomSeed() ); } diff --git a/maven-surefire-common/src/test/java/org/apache/maven/plugin/surefire/AbstractSurefireMojoJava7PlusTest.java b/maven-surefire-common/src/test/java/org/apache/maven/plugin/surefire/AbstractSurefireMojoJava7PlusTest.java index c9dbf5cde6..a1be9607c6 100644 --- a/maven-surefire-common/src/test/java/org/apache/maven/plugin/surefire/AbstractSurefireMojoJava7PlusTest.java +++ b/maven-surefire-common/src/test/java/org/apache/maven/plugin/surefire/AbstractSurefireMojoJava7PlusTest.java @@ -839,6 +839,12 @@ public void setRunOrder( String runOrder ) } + @Override + public String getMethodRunOrder() + { + return null; + }; + @Override public Long getRunOrderRandomSeed() { diff --git a/maven-surefire-common/src/test/java/org/apache/maven/plugin/surefire/AbstractSurefireMojoTest.java b/maven-surefire-common/src/test/java/org/apache/maven/plugin/surefire/AbstractSurefireMojoTest.java index 53e7fbafca..7b3c3206c0 100644 --- a/maven-surefire-common/src/test/java/org/apache/maven/plugin/surefire/AbstractSurefireMojoTest.java +++ b/maven-surefire-common/src/test/java/org/apache/maven/plugin/surefire/AbstractSurefireMojoTest.java @@ -2382,6 +2382,12 @@ public void setRunOrder( String runOrder ) } + @Override + public String getMethodRunOrder() + { + return null; + }; + @Override public Long getRunOrderRandomSeed() { diff --git a/maven-surefire-common/src/test/java/org/apache/maven/plugin/surefire/MojoMocklessTest.java b/maven-surefire-common/src/test/java/org/apache/maven/plugin/surefire/MojoMocklessTest.java index 1c9b89f6ae..3f630ba3ab 100644 --- a/maven-surefire-common/src/test/java/org/apache/maven/plugin/surefire/MojoMocklessTest.java +++ b/maven-surefire-common/src/test/java/org/apache/maven/plugin/surefire/MojoMocklessTest.java @@ -709,6 +709,12 @@ public void setRunOrder( String runOrder ) } + @Override + public String getMethodRunOrder() + { + return null; + }; + @Override public Long getRunOrderRandomSeed() { diff --git a/maven-surefire-common/src/test/java/org/apache/maven/plugin/surefire/booterclient/BooterDeserializerProviderConfigurationTest.java b/maven-surefire-common/src/test/java/org/apache/maven/plugin/surefire/booterclient/BooterDeserializerProviderConfigurationTest.java index 7a0839054a..d0b92452df 100644 --- a/maven-surefire-common/src/test/java/org/apache/maven/plugin/surefire/booterclient/BooterDeserializerProviderConfigurationTest.java +++ b/maven-surefire-common/src/test/java/org/apache/maven/plugin/surefire/booterclient/BooterDeserializerProviderConfigurationTest.java @@ -39,6 +39,7 @@ import org.apache.maven.surefire.api.testset.TestListResolver; import org.apache.maven.surefire.api.testset.TestRequest; import org.apache.maven.surefire.api.util.RunOrder; +import org.apache.maven.surefire.util.MethodRunOrder; import org.junit.After; import org.junit.Before; @@ -276,7 +277,8 @@ private ProviderConfiguration getTestProviderConfiguration( DirectoryScannerPara new TestRequest( getSuiteXmlFileStrings(), getTestSourceDirectory(), new TestListResolver( USER_REQUESTED_TEST + "#aUserRequestedTestMethod" ), RERUN_FAILING_TEST_COUNT ); - RunOrderParameters runOrderParameters = new RunOrderParameters( RunOrder.DEFAULT, null ); + RunOrderParameters runOrderParameters = new RunOrderParameters( RunOrder.DEFAULT, null, null, + MethodRunOrder.DEFAULT ); return new ProviderConfiguration( directoryScannerParameters, runOrderParameters, true, reporterConfiguration, new TestArtifactInfo( "5.0", "ABC" ), testSuiteDefinition, new HashMap(), TEST_TYPED, readTestsFromInStream, cli, 0, Shutdown.DEFAULT, 0 ); diff --git a/maven-surefire-common/src/test/java/org/apache/maven/plugin/surefire/booterclient/BooterDeserializerStartupConfigurationTest.java b/maven-surefire-common/src/test/java/org/apache/maven/plugin/surefire/booterclient/BooterDeserializerStartupConfigurationTest.java index 669e73cc3d..4fb64b54c6 100644 --- a/maven-surefire-common/src/test/java/org/apache/maven/plugin/surefire/booterclient/BooterDeserializerStartupConfigurationTest.java +++ b/maven-surefire-common/src/test/java/org/apache/maven/plugin/surefire/booterclient/BooterDeserializerStartupConfigurationTest.java @@ -38,6 +38,7 @@ import org.apache.maven.surefire.api.testset.TestListResolver; import org.apache.maven.surefire.api.testset.TestRequest; import org.apache.maven.surefire.api.util.RunOrder; +import org.apache.maven.surefire.util.MethodRunOrder; import org.junit.After; import org.junit.Before; @@ -197,7 +198,8 @@ private ProviderConfiguration getProviderConfiguration() new TestRequest( Arrays.asList( getSuiteXmlFileStrings() ), getTestSourceDirectory(), new TestListResolver( "aUserRequestedTest#aUserRequestedTestMethod" ) ); - RunOrderParameters runOrderParameters = new RunOrderParameters( RunOrder.DEFAULT, null ); + RunOrderParameters runOrderParameters = new RunOrderParameters( RunOrder.DEFAULT, null, null, + MethodRunOrder.DEFAULT ); return new ProviderConfiguration( directoryScannerParameters, runOrderParameters, true, reporterConfiguration, new TestArtifactInfo( "5.0", "ABC" ), testSuiteDefinition, new HashMap(), BooterDeserializerProviderConfigurationTest.TEST_TYPED, true, cli, 0, Shutdown.DEFAULT, 0 ); diff --git a/maven-surefire-plugin/src/main/java/org/apache/maven/plugin/surefire/SurefirePlugin.java b/maven-surefire-plugin/src/main/java/org/apache/maven/plugin/surefire/SurefirePlugin.java index 16ffaf9ffe..5e547d7b25 100644 --- a/maven-surefire-plugin/src/main/java/org/apache/maven/plugin/surefire/SurefirePlugin.java +++ b/maven-surefire-plugin/src/main/java/org/apache/maven/plugin/surefire/SurefirePlugin.java @@ -20,6 +20,9 @@ */ import java.io.File; +import java.io.IOException; +import java.nio.charset.Charset; +import java.nio.file.Files; import java.util.Arrays; import java.util.Collections; import java.util.List; @@ -302,6 +305,10 @@ public class SurefirePlugin @Parameter( property = "surefire.runOrder", defaultValue = "filesystem" ) private String runOrder; + //TODO docs + @Parameter( property = "surefire.methodRunOrder", defaultValue = "default" ) + private String methodRunOrder; + /** * Sets the random seed that will be used to order the tests if {@code surefire.runOrder} is set to {@code random}. *
@@ -629,6 +636,24 @@ public void setReportsDirectory( File reportsDirectory ) @Override public String getTest() { + File f = new File( test ); + if ( f.exists() && !f.isDirectory ( ) ) + { + try + { + List l = Files.readAllLines( f.toPath(), Charset.defaultCharset( ) ); + StringBuilder sb = new StringBuilder( ); + for ( String s : l ) + { + sb.append( s + "," ); + } + String s = sb.toString( ); + return s.substring( 0 , s.length( ) - 1 ); + } + catch ( IOException e ) + { + } + } return test; } @@ -832,6 +857,13 @@ public void setRunOrder( String runOrder ) this.runOrder = runOrder; } + + @Override + public String getMethodRunOrder() + { + return methodRunOrder; + } + @Override public Long getRunOrderRandomSeed() { diff --git a/pom.xml b/pom.xml index 3e36523d97..a5ae1cb5bd 100644 --- a/pom.xml +++ b/pom.xml @@ -502,7 +502,7 @@ org.codehaus.mojo animal-sniffer-maven-plugin - 1.17 + 1.18 maven-surefire-plugin diff --git a/surefire-api/src/main/java/org/apache/maven/surefire/api/testset/RunOrderParameters.java b/surefire-api/src/main/java/org/apache/maven/surefire/api/testset/RunOrderParameters.java index 07e3b3ea98..74138c77f7 100644 --- a/surefire-api/src/main/java/org/apache/maven/surefire/api/testset/RunOrderParameters.java +++ b/surefire-api/src/main/java/org/apache/maven/surefire/api/testset/RunOrderParameters.java @@ -21,6 +21,7 @@ import java.io.File; import org.apache.maven.surefire.api.util.RunOrder; +import org.apache.maven.surefire.util.MethodRunOrder; /** * @author Kristian Rosenvold @@ -31,20 +32,33 @@ public class RunOrderParameters private File runStatisticsFile; + private final MethodRunOrder methodRunOrder; + private Long runOrderRandomSeed; - public RunOrderParameters( RunOrder[] runOrder, File runStatisticsFile ) + public RunOrderParameters( RunOrder[] runOrder, File runStatisticsFile, MethodRunOrder methodRunOrder ) { this.runOrder = runOrder; this.runStatisticsFile = runStatisticsFile; + this.methodRunOrder = methodRunOrder; this.runOrderRandomSeed = null; } - public RunOrderParameters( String runOrder, File runStatisticsFile ) + public RunOrderParameters( RunOrder[] runOrder, File runStatisticsFile, Long runOrderRandomSeed, + MethodRunOrder methodRunOrder ) + { + this.runOrder = runOrder; + this.runStatisticsFile = runStatisticsFile; + this.methodRunOrder = methodRunOrder; + this.runOrderRandomSeed = runOrderRandomSeed; + } + + public RunOrderParameters( String runOrder, File runStatisticsFile, Long runOrderRandomSeed, String methodRunOrder ) { this.runOrder = runOrder == null ? RunOrder.DEFAULT : RunOrder.valueOfMulti( runOrder ); this.runStatisticsFile = runStatisticsFile; - this.runOrderRandomSeed = null; + this.methodRunOrder = MethodRunOrder.valueOf( methodRunOrder ); + this.runOrderRandomSeed = runOrderRandomSeed; } public RunOrderParameters( RunOrder[] runOrder, File runStatisticsFile, Long runOrderRandomSeed ) @@ -52,6 +66,7 @@ public RunOrderParameters( RunOrder[] runOrder, File runStatisticsFile, Long run this.runOrder = runOrder; this.runStatisticsFile = runStatisticsFile; this.runOrderRandomSeed = runOrderRandomSeed; + this.methodRunOrder = MethodRunOrder.DEFAULT; } public RunOrderParameters( String runOrder, File runStatisticsFile, Long runOrderRandomSeed ) @@ -59,11 +74,28 @@ public RunOrderParameters( String runOrder, File runStatisticsFile, Long runOrde this.runOrder = runOrder == null ? RunOrder.DEFAULT : RunOrder.valueOfMulti( runOrder ); this.runStatisticsFile = runStatisticsFile; this.runOrderRandomSeed = runOrderRandomSeed; + this.methodRunOrder = MethodRunOrder.DEFAULT; + } + + public RunOrderParameters( RunOrder[] runOrder, File runStatisticsFile ) + { + this.runOrder = runOrder; + this.runStatisticsFile = runStatisticsFile; + this.runOrderRandomSeed = null; + this.methodRunOrder = MethodRunOrder.DEFAULT; + } + + public RunOrderParameters( String runOrder, File runStatisticsFile ) + { + this.runOrder = runOrder == null ? RunOrder.DEFAULT : RunOrder.valueOfMulti( runOrder ); + this.runStatisticsFile = runStatisticsFile; + this.runOrderRandomSeed = null; + this.methodRunOrder = MethodRunOrder.DEFAULT; } public static RunOrderParameters alphabetical() { - return new RunOrderParameters( new RunOrder[]{ RunOrder.ALPHABETICAL }, null ); + return new RunOrderParameters( new RunOrder[]{ RunOrder.ALPHABETICAL }, null, null, MethodRunOrder.DEFAULT ); } public RunOrder[] getRunOrder() @@ -86,4 +118,9 @@ public File getRunStatisticsFile() return runStatisticsFile; } + public MethodRunOrder getMethodRunOrder() + { + return methodRunOrder; + } + } diff --git a/surefire-api/src/main/java/org/apache/maven/surefire/api/util/DefaultRunOrderCalculator.java b/surefire-api/src/main/java/org/apache/maven/surefire/api/util/DefaultRunOrderCalculator.java index 8ced06afa5..c2b14ad96e 100644 --- a/surefire-api/src/main/java/org/apache/maven/surefire/api/util/DefaultRunOrderCalculator.java +++ b/surefire-api/src/main/java/org/apache/maven/surefire/api/util/DefaultRunOrderCalculator.java @@ -21,12 +21,20 @@ import org.apache.maven.surefire.api.runorder.RunEntryStatisticsMap; import org.apache.maven.surefire.api.testset.RunOrderParameters; +import org.apache.maven.surefire.util.MethodRunOrder; +import java.io.File; +import java.io.IOException; +import java.nio.charset.Charset; +import java.nio.file.Files; import java.util.ArrayList; import java.util.Calendar; import java.util.Collections; import java.util.Comparator; +import java.util.HashMap; +import java.util.LinkedHashMap; import java.util.LinkedHashSet; +import java.util.LinkedList; import java.util.List; import java.util.Random; @@ -78,9 +86,159 @@ public TestsToRun orderTestClasses( TestsToRun scannedClasses ) return new TestsToRun( new LinkedHashSet<>( result ) ); } + @Override + public Comparator comparatorForTestMethods() + { + MethodRunOrder order = runOrderParameters.getMethodRunOrder(); + if ( MethodRunOrder.DEFAULT.equals( order ) ) + { + return null; + } + else if ( MethodRunOrder.ALPHABETICAL.equals( order ) ) + { + return new Comparator() + { + @Override + public int compare( String o1, String o2 ) + { + return o1.compareTo( o2 ); + } + }; + } + else if ( MethodRunOrder.REVERSE_ALPHABETICAL.equals( order ) ) + { + return new Comparator() + { + @Override + public int compare( String o1, String o2 ) + { + return o2.compareTo( o1 ); + } + }; + } + else if ( MethodRunOrder.RANDOM.equals( order ) ) + { + return new Comparator() + { + HashMap randomVals = new HashMap<>(); + + private int getRandom( String obj ) + { + if ( !randomVals.containsKey( obj ) ) + { + randomVals.put( obj, random.nextInt() ); + } + return randomVals.get( obj ); + } + + @Override + public int compare( String o1, String o2 ) + { + int i1 = getRandom( o1 ); + int i2 = getRandom( o2 ); + return ( i1 > i2 ? 1 : -1 ); + } + }; + } + else if ( MethodRunOrder.FLAKY_FINDING.equals( order ) ) + { + String orderParam = parseFlakyTestOrder( System.getProperty( "test" ) ); + if ( orderParam == null ) + { + throw new IllegalStateException( "Please set system property -Dtest to use fixed order" ); + } + final LinkedHashMap> orders = new LinkedHashMap<>(); + for ( String s : orderParam.split( "," ) ) + { + String[] nameSplit = s.split( "#" ); + String className = nameSplit[0]; + String testName = nameSplit[1]; + String parenName = testName + "(" + className + ")"; + addTestToOrders( className, orders, parenName ); + } + return new Comparator() + { + + @Override + public int compare( String o1, String o2 ) + { + String className1 = o1; + if ( o1.contains( "(" ) ) + { + String[] nameSplit1 = o1.split( "\\(" ); + className1 = nameSplit1[1].substring( 0, nameSplit1[1].length() - 1 ); + addTestToOrders( className1, orders, o1 ); + } + + String className2 = o2; + if ( o2.contains( "(" ) ) + { + String[] nameSplit2 = o2.split( "\\(" ); + className2 = nameSplit2[1].substring( 0, nameSplit2[1].length() - 1 ); + addTestToOrders( className2, orders, o2 ); + } + + if ( ! className2.equals( className1 ) ) + { + List classOrders = new ArrayList( orders.keySet() ); + if ( classOrders.indexOf( className1 ) < classOrders.indexOf( className2 ) ) + { + return -1; + } + else + { + return 1; + } + } + else + { + List testOrders = orders.get( className2 ); + if ( testOrders.indexOf( o1 ) < testOrders.indexOf( o2 ) ) + { + return -1; + } + else + { + return 1; + } + } + } + }; + } + else + { + throw new UnsupportedOperationException( "Unsupported method run order: " + order.name() ); + } + } + + public void addTestToOrders( String className, LinkedHashMap> orders, String parenName ) + { + List classOrders = orders.get( className ); + if ( classOrders == null ) + { + classOrders = new ArrayList(); + } + if ( ! classOrders.contains( parenName ) ) + { + classOrders.add( parenName ); + } + if ( ! orders.containsKey( className ) ) + { + orders.put( className, classOrders ); + } + } + private void orderTestClasses( List> testClasses, RunOrder runOrder ) { - if ( RunOrder.RANDOM.equals( runOrder ) ) + if ( System.getProperty( "surefire.methodRunOrder" ) != null + && System.getProperty( "surefire.methodRunOrder" ).toLowerCase().equals( "fixed" ) ) + { + List> sorted + = sortClassesBySpecifiedOrder( testClasses, parseFlakyTestOrder( System.getProperty( "test" ) ) ); + testClasses.clear(); + testClasses.addAll( sorted ); + } + else if ( RunOrder.RANDOM.equals( runOrder ) ) { Collections.shuffle( testClasses, random ); } @@ -106,6 +264,52 @@ else if ( sortOrder != null ) } } + private String parseFlakyTestOrder( String s ) + { + if ( s != null && s != "" ) + { + File f = new File( s ); + if ( f.exists() && !f.isDirectory ( ) ) + { + try + { + List l = Files.readAllLines( f.toPath(), Charset.defaultCharset( ) ); + StringBuilder sb = new StringBuilder( ); + for ( String sd : l ) + { + sb.append( sd + "," ); + } + String sd = sb.toString( ); + return sd.substring( 0 , sd.length( ) - 1 ); + } + catch ( IOException e ) + { + } + } + } + return s; + } + + private List> sortClassesBySpecifiedOrder( List> testClasses, String flakyTestOrder ) + { + HashMap> classes = new HashMap<>(); + for ( Class each : testClasses ) + { + classes.put( each.getName(), each ); + } + LinkedList> ret = new LinkedList<>(); + for ( String s : flakyTestOrder.split( "," ) ) + { + String testClass = s.substring( 0, s.indexOf( '#' ) ); + Class c = classes.remove( testClass ); + if ( c != null ) + { + ret.add( c ); + } + } + return ret; + } + private Comparator getSortOrderComparator( RunOrder runOrder ) { if ( RunOrder.ALPHABETICAL.equals( runOrder ) ) diff --git a/surefire-api/src/main/java/org/apache/maven/surefire/api/util/RunOrderCalculator.java b/surefire-api/src/main/java/org/apache/maven/surefire/api/util/RunOrderCalculator.java index a9f99939ed..2c58002089 100644 --- a/surefire-api/src/main/java/org/apache/maven/surefire/api/util/RunOrderCalculator.java +++ b/surefire-api/src/main/java/org/apache/maven/surefire/api/util/RunOrderCalculator.java @@ -19,10 +19,14 @@ * under the License. */ +import java.util.Comparator; + /** * @author Kristian Rosenvold */ public interface RunOrderCalculator { TestsToRun orderTestClasses( TestsToRun scannedClasses ); + + Comparator comparatorForTestMethods(); } diff --git a/surefire-api/src/main/java/org/apache/maven/surefire/util/MethodRunOrder.java b/surefire-api/src/main/java/org/apache/maven/surefire/util/MethodRunOrder.java new file mode 100644 index 0000000000..d41da320fb --- /dev/null +++ b/surefire-api/src/main/java/org/apache/maven/surefire/util/MethodRunOrder.java @@ -0,0 +1,100 @@ +package org.apache.maven.surefire.util; + +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +/** + * TODO docs + */ +public class MethodRunOrder +{ + + public static final MethodRunOrder ALPHABETICAL = new MethodRunOrder( "alphabetical" ); + public static final MethodRunOrder REVERSE_ALPHABETICAL = new MethodRunOrder( "reversealphabetical" ); + public static final MethodRunOrder RANDOM = new MethodRunOrder( "random" ); + public static final MethodRunOrder FLAKY_FINDING = new MethodRunOrder( "fixed" ); + public static final MethodRunOrder DEFAULT = new MethodRunOrder( "default" ); + + private final String name; + + MethodRunOrder( String name ) + { + this.name = name; + } + + private static MethodRunOrder[] values() + { + return new MethodRunOrder[] {ALPHABETICAL, REVERSE_ALPHABETICAL, RANDOM, FLAKY_FINDING, DEFAULT}; + } + + public boolean matches( String anotherName ) + { + return name.equalsIgnoreCase( anotherName ); + } + + public static MethodRunOrder valueOf( String name ) + { + if ( name == null ) + { + return null; + } + else + { + for ( MethodRunOrder each : values() ) + { + if ( each.matches( name ) ) + { + return each; + } + } + } + String errorMessage = createMessageForMissingRunOrder( name ); + throw new IllegalArgumentException( errorMessage ); + + } + + private static String createMessageForMissingRunOrder( String name ) + { + MethodRunOrder[] methodRunOrders = values(); + StringBuilder message = new StringBuilder( "There's no MethodRunOrder with the name " ); + message.append( name ); + message.append( ". Please use one of the following RunOrders: " ); + for ( int i = 0; i < methodRunOrders.length; i++ ) + { + if ( i != 0 ) + { + message.append( ", " ); + } + message.append( methodRunOrders[i] ); + } + message.append( '.' ); + return message.toString(); + } + + @Override + public String toString() + { + return name; + } + + public String name() + { + return name; + } +} diff --git a/surefire-booter/src/main/java/org/apache/maven/surefire/booter/BooterConstants.java b/surefire-booter/src/main/java/org/apache/maven/surefire/booter/BooterConstants.java index dcf73615da..ecbfa87e3c 100644 --- a/surefire-booter/src/main/java/org/apache/maven/surefire/booter/BooterConstants.java +++ b/surefire-booter/src/main/java/org/apache/maven/surefire/booter/BooterConstants.java @@ -47,6 +47,7 @@ private BooterConstants() public static final String RUN_ORDER = "runOrder"; public static final String RUN_ORDER_RANDOM_SEED = "runOrderRandomSeed"; public static final String RUN_STATISTICS_FILE = "runStatisticsFile"; + public static final String METHOD_RUN_ORDER = "methodRunOrder"; public static final String TEST_SUITE_XML_FILES = "testSuiteXmlFiles"; public static final String PROVIDER_CONFIGURATION = "providerConfiguration"; public static final String FORKTESTSET = "forkTestSet"; diff --git a/surefire-booter/src/main/java/org/apache/maven/surefire/booter/BooterDeserializer.java b/surefire-booter/src/main/java/org/apache/maven/surefire/booter/BooterDeserializer.java index ca5328e52f..7689b9589d 100644 --- a/surefire-booter/src/main/java/org/apache/maven/surefire/booter/BooterDeserializer.java +++ b/surefire-booter/src/main/java/org/apache/maven/surefire/booter/BooterDeserializer.java @@ -109,6 +109,7 @@ public ProviderConfiguration deserialize() final String runOrder = properties.getProperty( RUN_ORDER ); final Long runOrderRandomSeed = properties.getLongProperty( RUN_ORDER_RANDOM_SEED ); final String runStatisticsFile = properties.getProperty( RUN_STATISTICS_FILE ); + final String methodRunOrder = properties.getProperty( METHOD_RUN_ORDER ); final int rerunFailingTestsCount = properties.getIntProperty( RERUN_FAILING_TESTS_COUNT ); @@ -118,7 +119,7 @@ public ProviderConfiguration deserialize() RunOrderParameters runOrderParameters = new RunOrderParameters( runOrder, runStatisticsFile == null ? null : new File( runStatisticsFile ), - runOrderRandomSeed ); + runOrderRandomSeed, methodRunOrder ); TestArtifactInfo testNg = new TestArtifactInfo( testNgVersion, testArtifactClassifier ); TestRequest testSuiteDefinition = diff --git a/surefire-booter/src/main/java/org/apache/maven/surefire/booter/SurefireReflector.java b/surefire-booter/src/main/java/org/apache/maven/surefire/booter/SurefireReflector.java index d4f459a135..ddd7277806 100644 --- a/surefire-booter/src/main/java/org/apache/maven/surefire/booter/SurefireReflector.java +++ b/surefire-booter/src/main/java/org/apache/maven/surefire/booter/SurefireReflector.java @@ -182,7 +182,8 @@ private Object createRunOrderParameters( RunOrderParameters runOrderParameters ) Constructor constructor = getConstructor( this.runOrderParameters, arguments ); File runStatisticsFile = runOrderParameters.getRunStatisticsFile(); return newInstance( constructor, RunOrder.asString( runOrderParameters.getRunOrder() ), runStatisticsFile, - runOrderParameters.getRunOrderRandomSeed() ); + runOrderParameters.getRunOrderRandomSeed(), + runOrderParameters.getMethodRunOrder().toString() ); } private Object createTestArtifactInfo( TestArtifactInfo testArtifactInfo ) diff --git a/surefire-booter/src/test/java/org/apache/maven/surefire/booter/SurefireReflectorTest.java b/surefire-booter/src/test/java/org/apache/maven/surefire/booter/SurefireReflectorTest.java index 88fd58361d..e301a6fb9d 100644 --- a/surefire-booter/src/test/java/org/apache/maven/surefire/booter/SurefireReflectorTest.java +++ b/surefire-booter/src/test/java/org/apache/maven/surefire/booter/SurefireReflectorTest.java @@ -33,6 +33,7 @@ import org.apache.maven.surefire.api.testset.TestListResolver; import org.apache.maven.surefire.api.testset.TestRequest; import org.apache.maven.surefire.api.util.RunOrder; +import org.apache.maven.surefire.util.MethodRunOrder; import java.io.File; import java.lang.reflect.Method; @@ -114,7 +115,8 @@ public void testRunOrderParameters() SurefireReflector surefireReflector = getReflector(); Object foo = getFoo(); - RunOrderParameters runOrderParameters = new RunOrderParameters( RunOrder.DEFAULT, new File( "." ) ); + RunOrderParameters runOrderParameters = new RunOrderParameters( RunOrder.DEFAULT, new File( "." ), + 0L, MethodRunOrder.DEFAULT ); surefireReflector.setRunOrderParameters( foo, runOrderParameters ); assertTrue( isCalled( foo ) ); } diff --git a/surefire-its/src/test/resources/surefire-818-ignored-tests-on-npe/pom.xml b/surefire-its/src/test/resources/surefire-818-ignored-tests-on-npe/pom.xml index cef444e3f2..cd4862d95b 100644 --- a/surefire-its/src/test/resources/surefire-818-ignored-tests-on-npe/pom.xml +++ b/surefire-its/src/test/resources/surefire-818-ignored-tests-on-npe/pom.xml @@ -31,6 +31,9 @@ maven-surefire-plugin ${surefire.version} + + true + diff --git a/surefire-providers/surefire-junit4/src/main/java/org/apache/maven/surefire/junit4/JUnit4Provider.java b/surefire-providers/surefire-junit4/src/main/java/org/apache/maven/surefire/junit4/JUnit4Provider.java index 3f4b095b76..7b345820f1 100644 --- a/surefire-providers/surefire-junit4/src/main/java/org/apache/maven/surefire/junit4/JUnit4Provider.java +++ b/surefire-providers/surefire-junit4/src/main/java/org/apache/maven/surefire/junit4/JUnit4Provider.java @@ -48,6 +48,7 @@ import org.junit.runner.notification.StoppedByUserException; import java.util.Collection; +import java.util.Comparator; import java.util.Set; import static java.lang.reflect.Modifier.isAbstract; @@ -269,7 +270,8 @@ private void executeWithRerun( Class clazz, Notifier notifier ) try { notifier.asFailFast( isFailFast() ); - execute( clazz, notifier, hasMethodFilter ? createMethodFilter() : null ); + execute( clazz, notifier, hasMethodFilter ? createMethodFilter() : null, + runOrderCalculator.comparatorForTestMethods() ); } finally { @@ -286,7 +288,8 @@ private void executeWithRerun( Class clazz, Notifier notifier ) Set failures = generateFailingTestDescriptions( failureListener.getAllFailures() ); failureListener.reset(); Filter failureDescriptionFilter = createMatchAnyDescriptionFilter( failures ); - execute( clazz, rerunNotifier, failureDescriptionFilter ); + execute( clazz, rerunNotifier, failureDescriptionFilter, + runOrderCalculator.comparatorForTestMethods() ); } } } @@ -348,12 +351,24 @@ private static boolean isJUnit4UpgradeCheck() return System.getProperty( "surefire.junit4.upgradecheck" ) != null; } - private static void execute( Class testClass, Notifier notifier, Filter filter ) + private static void execute( Class testClass, Notifier notifier, Filter filter, + final Comparator runOrderComparator ) { final int classModifiers = testClass.getModifiers(); if ( !isAbstract( classModifiers ) && !isInterface( classModifiers ) ) { Request request = aClass( testClass ); + if ( runOrderComparator != null ) + { + request = request.sortWith( new Comparator() + { + @Override + public int compare( Description o1, Description o2 ) + { + return runOrderComparator.compare( o1.toString(), o2.toString() ); + } + } ); + } if ( filter != null ) { request = request.filterWith( filter ); diff --git a/surefire-providers/surefire-junit47/src/main/java/org/apache/maven/surefire/junitcore/JUnitCoreProvider.java b/surefire-providers/surefire-junit47/src/main/java/org/apache/maven/surefire/junitcore/JUnitCoreProvider.java index 6877b00b28..e842c712d0 100644 --- a/surefire-providers/surefire-junit47/src/main/java/org/apache/maven/surefire/junitcore/JUnitCoreProvider.java +++ b/surefire-providers/surefire-junit47/src/main/java/org/apache/maven/surefire/junitcore/JUnitCoreProvider.java @@ -145,7 +145,8 @@ public RunResult invoke( Object forkTestSet ) try { - JUnitCoreWrapper core = new JUnitCoreWrapper( notifier, jUnitCoreParameters, consoleStream ); + JUnitCoreWrapper core = new JUnitCoreWrapper( notifier, jUnitCoreParameters, consoleStream, + runOrderCalculator ); if ( commandsReader != null ) { @@ -162,7 +163,8 @@ public RunResult invoke( Object forkTestSet ) { Notifier rerunNotifier = pureNotifier(); notifier.copyListenersTo( rerunNotifier ); - JUnitCoreWrapper rerunCore = new JUnitCoreWrapper( rerunNotifier, jUnitCoreParameters, consoleStream ); + JUnitCoreWrapper rerunCore = new JUnitCoreWrapper( rerunNotifier, jUnitCoreParameters, consoleStream, + runOrderCalculator ); for ( int i = 0; i < rerunFailingTestsCount && !testFailureListener.getAllFailures().isEmpty(); i++ ) { Set failures = generateFailingTestDescriptions( testFailureListener.getAllFailures() ); diff --git a/surefire-providers/surefire-junit47/src/main/java/org/apache/maven/surefire/junitcore/JUnitCoreWrapper.java b/surefire-providers/surefire-junit47/src/main/java/org/apache/maven/surefire/junitcore/JUnitCoreWrapper.java index 60ba4708bb..bf45ba5357 100644 --- a/surefire-providers/surefire-junit47/src/main/java/org/apache/maven/surefire/junitcore/JUnitCoreWrapper.java +++ b/surefire-providers/surefire-junit47/src/main/java/org/apache/maven/surefire/junitcore/JUnitCoreWrapper.java @@ -24,6 +24,7 @@ import org.apache.maven.surefire.junitcore.pc.ParallelComputerBuilder; import org.apache.maven.surefire.api.report.ConsoleStream; import org.apache.maven.surefire.api.testset.TestSetFailedException; +import org.apache.maven.surefire.api.util.RunOrderCalculator; import org.apache.maven.surefire.api.util.TestsToRun; import org.junit.Ignore; import org.junit.runner.Computer; @@ -36,6 +37,7 @@ import java.util.Collection; import java.util.Collections; +import java.util.Comparator; import java.util.Iterator; import java.util.Queue; @@ -55,12 +57,15 @@ final class JUnitCoreWrapper private final Notifier notifier; private final JUnitCoreParameters jUnitCoreParameters; private final ConsoleStream consoleStream; + private final RunOrderCalculator runOrderCalculator; - JUnitCoreWrapper( Notifier notifier, JUnitCoreParameters jUnitCoreParameters, ConsoleStream consoleStream ) + JUnitCoreWrapper( Notifier notifier, JUnitCoreParameters jUnitCoreParameters, ConsoleStream consoleStream, + RunOrderCalculator runOrderCalculator ) { this.notifier = notifier; this.jUnitCoreParameters = jUnitCoreParameters; this.consoleStream = consoleStream; + this.runOrderCalculator = runOrderCalculator; } void execute( TestsToRun testsToRun, Filter filter ) @@ -124,6 +129,18 @@ private void createRequestAndRun( Filter filter, Computer computer, JUnitCore ju throws TestSetFailedException { Request req = classes( computer, classesToRun ); + final Comparator testOrderComparator = runOrderCalculator.comparatorForTestMethods(); + if ( testOrderComparator != null ) + { + req = req.sortWith( new Comparator() + { + @Override + public int compare( Description o1, Description o2 ) + { + return testOrderComparator.compare( o1.toString(), o2.toString() ); + } + } ); + } if ( filter != null ) { req = new FilteringRequest( req, filter ); diff --git a/surefire-providers/surefire-junit47/src/test/java/org/apache/maven/surefire/junitcore/Surefire746Test.java b/surefire-providers/surefire-junit47/src/test/java/org/apache/maven/surefire/junitcore/Surefire746Test.java index 38690ae156..b5e677de74 100644 --- a/surefire-providers/surefire-junit47/src/test/java/org/apache/maven/surefire/junitcore/Surefire746Test.java +++ b/surefire-providers/surefire-junit47/src/test/java/org/apache/maven/surefire/junitcore/Surefire746Test.java @@ -53,7 +53,9 @@ import org.apache.maven.surefire.api.report.ReporterFactory; import org.apache.maven.surefire.api.report.RunListener; import org.apache.maven.surefire.api.suite.RunResult; +import org.apache.maven.surefire.api.testset.RunOrderParameters; import org.apache.maven.surefire.api.testset.TestSetFailedException; +import org.apache.maven.surefire.api.util.DefaultRunOrderCalculator; import org.apache.maven.surefire.api.util.TestsToRun; import org.junit.Rule; @@ -137,7 +139,9 @@ public void surefireIsConfused_ByMultipleIgnore_OnClassLevel() throws Exception exception.expect( TestSetFailedException.class ); JUnit4RunListener dummy = new JUnit4RunListener( new MockReporter() ); new JUnitCoreWrapper( new Notifier( dummy, 0 ), jUnitCoreParameters, - new DefaultDirectConsoleReporter( System.out ) ).execute( testsToRun, customRunListeners, null ); + new DefaultDirectConsoleReporter( System.out ), + new DefaultRunOrderCalculator( RunOrderParameters.alphabetical(), 1 ) ). + execute( testsToRun, customRunListeners, null ); } finally {