From 8a6b33453d3114e755ad9f0776bd8f5ca7bf3bc0 Mon Sep 17 00:00:00 2001 From: Davide Angelocola Date: Wed, 3 Apr 2019 00:11:24 +0200 Subject: [PATCH 01/24] [SUREFIRE-1432] Add extension interface with two implementations for trimStackTrace --- .../org/apache/maven/plugin/surefire/AbstractSurefireMojo.java | 2 +- .../test/resources/surefire-818-ignored-tests-on-npe/pom.xml | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/maven-surefire-common/src/main/java/org/apache/maven/plugin/surefire/AbstractSurefireMojo.java b/maven-surefire-common/src/main/java/org/apache/maven/plugin/surefire/AbstractSurefireMojo.java index 856a527f70..829148b17e 100644 --- a/maven-surefire-common/src/main/java/org/apache/maven/plugin/surefire/AbstractSurefireMojo.java +++ b/maven-surefire-common/src/main/java/org/apache/maven/plugin/surefire/AbstractSurefireMojo.java @@ -654,7 +654,7 @@ public abstract class AbstractSurefireMojo * * @since 2.2 */ - @Parameter( property = "trimStackTrace", defaultValue = "true" ) + @Parameter( property = "trimStackTrace", defaultValue = "false" ) private boolean trimStackTrace; /** diff --git a/surefire-its/src/test/resources/surefire-818-ignored-tests-on-npe/pom.xml b/surefire-its/src/test/resources/surefire-818-ignored-tests-on-npe/pom.xml index cef444e3f2..cd4862d95b 100644 --- a/surefire-its/src/test/resources/surefire-818-ignored-tests-on-npe/pom.xml +++ b/surefire-its/src/test/resources/surefire-818-ignored-tests-on-npe/pom.xml @@ -31,6 +31,9 @@ maven-surefire-plugin ${surefire.version} + + true + From 125c043ee58ce6167d74ea285557304db836d6ff Mon Sep 17 00:00:00 2001 From: Jonathan Bell Date: Sat, 11 Apr 2020 12:32:39 -0400 Subject: [PATCH 02/24] adds a new configuration option, 'methodRunOrder' (or -Dsurefire.methodRunOrder on command line) to choose order of methods. possible values include: unsorted (default, however junit would do it otherwise), alphabetical, reverse_alphabetical and random. New custom orders can be implemented in DefaultRunOrderCalculator class. Supports Junit4 and JUnit47 providers. --- .../plugin/failsafe/IntegrationTestMojo.java | 26 +++++ .../plugin/surefire/AbstractSurefireMojo.java | 25 ++++- .../booterclient/BooterSerializer.java | 4 + .../AbstractSurefireMojoJava7PlusTest.java | 18 ++++ .../surefire/AbstractSurefireMojoTest.java | 18 ++++ .../plugin/surefire/MojoMocklessTest.java | 18 ++++ ...DeserializerProviderConfigurationTest.java | 4 +- ...rDeserializerStartupConfigurationTest.java | 4 +- .../maven/plugin/surefire/SurefirePlugin.java | 27 +++++ .../surefire/booter/SurefireReflector.java | 5 +- .../surefire/testset/RunOrderParameters.java | 29 ++++- .../util/DefaultRunOrderCalculator.java | 72 ++++++++++++- .../maven/surefire/util/MethodRunOrder.java | 100 ++++++++++++++++++ .../surefire/util/RunOrderCalculator.java | 4 + .../booter/SurefireReflectorTest.java | 4 +- .../surefire/booter/BooterConstants.java | 2 + .../surefire/booter/BooterDeserializer.java | 5 +- .../maven/surefire/junit4/JUnit4Provider.java | 21 +++- .../surefire/junitcore/JUnitCoreProvider.java | 6 +- .../surefire/junitcore/JUnitCoreWrapper.java | 19 +++- .../surefire/junitcore/Surefire746Test.java | 6 +- 21 files changed, 398 insertions(+), 19 deletions(-) create mode 100644 surefire-api/src/main/java/org/apache/maven/surefire/util/MethodRunOrder.java diff --git a/maven-failsafe-plugin/src/main/java/org/apache/maven/plugin/failsafe/IntegrationTestMojo.java b/maven-failsafe-plugin/src/main/java/org/apache/maven/plugin/failsafe/IntegrationTestMojo.java index e465bb28ca..7e0818a2b9 100644 --- a/maven-failsafe-plugin/src/main/java/org/apache/maven/plugin/failsafe/IntegrationTestMojo.java +++ b/maven-failsafe-plugin/src/main/java/org/apache/maven/plugin/failsafe/IntegrationTestMojo.java @@ -309,6 +309,14 @@ public class IntegrationTestMojo @Parameter( property = "failsafe.runOrder", defaultValue = "filesystem" ) private String runOrder; + //TODO docs + @Parameter( property = "failsafe.methodRunOrder", defaultValue = "default" ) + private String methodRunOrder; + + @Parameter( property = "failsafe.seed" ) + private long randomSeed; + + /** * A file containing include patterns, each in a next line. Blank lines, or lines starting with # are ignored. * If {@code includes} are also specified, these patterns are appended. Example with path, simple and regex @@ -875,6 +883,24 @@ public void setRunOrder( String runOrder ) this.runOrder = runOrder; } + @Override + public long getRandomSeed() + { + return randomSeed; + } + + @Override + public void setRandomSeed( long randomSeed ) + { + this.randomSeed = randomSeed; + } + + @Override + public String getMethodRunOrder() + { + return methodRunOrder; + } + @Override public File getIncludesFile() { diff --git a/maven-surefire-common/src/main/java/org/apache/maven/plugin/surefire/AbstractSurefireMojo.java b/maven-surefire-common/src/main/java/org/apache/maven/plugin/surefire/AbstractSurefireMojo.java index a8c1927852..f55b85343c 100644 --- a/maven-surefire-common/src/main/java/org/apache/maven/plugin/surefire/AbstractSurefireMojo.java +++ b/maven-surefire-common/src/main/java/org/apache/maven/plugin/surefire/AbstractSurefireMojo.java @@ -83,6 +83,7 @@ import org.apache.maven.surefire.testset.TestRequest; import org.apache.maven.surefire.testset.TestSetFailedException; import org.apache.maven.surefire.util.DefaultScanResult; +import org.apache.maven.surefire.util.MethodRunOrder; import org.apache.maven.surefire.util.RunOrder; import org.apache.maven.toolchain.DefaultToolchain; import org.apache.maven.toolchain.Toolchain; @@ -820,6 +821,12 @@ public abstract class AbstractSurefireMojo public abstract void setRunOrder( String runOrder ); + public abstract String getMethodRunOrder(); + + public abstract long getRandomSeed(); + + public abstract void setRandomSeed( long seed ); + protected abstract void handleSummary( RunResult summary, Exception firstForkException ) throws MojoExecutionException, MojoFailureException; @@ -1039,6 +1046,7 @@ boolean verifyParameters() warnIfWrongShutdownValue(); warnIfNotApplicableSkipAfterFailureCount(); warnIfIllegalTempDir(); + printDefaultSeedIfNecessary(); } return true; } @@ -1183,7 +1191,8 @@ private RunResult executeProvider( @Nonnull ProviderInfo provider, @Nonnull Defa ClassLoaderConfiguration classLoaderConfiguration = getClassLoaderConfiguration(); provider.addProviderProperties(); RunOrderParameters runOrderParameters = - new RunOrderParameters( getRunOrder(), getStatisticsFile( getConfigChecksum() ) ); + new RunOrderParameters( getRunOrder(), getStatisticsFile( getConfigChecksum() ), getRandomSeed(), + getMethodRunOrder() ); Platform platform = PLATFORM.withJdkExecAttributesForTests( getEffectiveJvm() ); if ( isNotForking() ) @@ -2553,6 +2562,7 @@ private String getConfigChecksum() checksum.add( getObjectFactory() ); checksum.add( getFailIfNoTests() ); checksum.add( getRunOrder() ); + checksum.add( getMethodRunOrder().toString() ); checksum.add( getDependenciesToScan() ); checksum.add( getForkedProcessExitTimeoutInSeconds() ); checksum.add( getRerunFailingTestsCount() ); @@ -2860,6 +2870,19 @@ private void warnIfIllegalTempDir() throws MojoFailureException } } + private void printDefaultSeedIfNecessary() + { + if ( getRandomSeed() == 0 && ( getRunOrder().equals( RunOrder.RANDOM.name() ) || getMethodRunOrder().equals( + MethodRunOrder.RANDOM.name() ) ) ) + { + setRandomSeed( System.nanoTime() ); + getConsoleLogger().info( + "Tests will run in random order. To reproduce ordering use flag -D" + + getPluginName() + ".seed=" + getRandomSeed() ); + } + + } + final class TestNgProviderInfo implements ProviderInfo { diff --git a/maven-surefire-common/src/main/java/org/apache/maven/plugin/surefire/booterclient/BooterSerializer.java b/maven-surefire-common/src/main/java/org/apache/maven/plugin/surefire/booterclient/BooterSerializer.java index 7eacb7485b..77673f9472 100644 --- a/maven-surefire-common/src/main/java/org/apache/maven/plugin/surefire/booterclient/BooterSerializer.java +++ b/maven-surefire-common/src/main/java/org/apache/maven/plugin/surefire/booterclient/BooterSerializer.java @@ -56,11 +56,13 @@ import static org.apache.maven.surefire.booter.BooterConstants.PLUGIN_PID; import static org.apache.maven.surefire.booter.BooterConstants.PROCESS_CHECKER; import static org.apache.maven.surefire.booter.BooterConstants.PROVIDER_CONFIGURATION; +import static org.apache.maven.surefire.booter.BooterConstants.RANDOM_SEED; import static org.apache.maven.surefire.booter.BooterConstants.REPORTSDIRECTORY; import static org.apache.maven.surefire.booter.BooterConstants.REQUESTEDTEST; import static org.apache.maven.surefire.booter.BooterConstants.RERUN_FAILING_TESTS_COUNT; import static org.apache.maven.surefire.booter.BooterConstants.RUN_ORDER; import static org.apache.maven.surefire.booter.BooterConstants.RUN_STATISTICS_FILE; +import static org.apache.maven.surefire.booter.BooterConstants.METHOD_RUN_ORDER; import static org.apache.maven.surefire.booter.BooterConstants.SHUTDOWN; import static org.apache.maven.surefire.booter.BooterConstants.SOURCE_DIRECTORY; import static org.apache.maven.surefire.booter.BooterConstants.SPECIFIC_TEST_PROPERTY_PREFIX; @@ -160,6 +162,8 @@ File serialize( KeyValueSource sourceProperties, ProviderConfiguration providerC { properties.setProperty( RUN_ORDER, RunOrder.asString( runOrderParameters.getRunOrder() ) ); properties.setProperty( RUN_STATISTICS_FILE, runOrderParameters.getRunStatisticsFile() ); + properties.setProperty( METHOD_RUN_ORDER, runOrderParameters.getMethodRunOrder().toString() ); + properties.setProperty( RANDOM_SEED, runOrderParameters.getRandomSeed() ); } ReporterConfiguration reporterConfiguration = providerConfiguration.getReporterConfiguration(); diff --git a/maven-surefire-common/src/test/java/org/apache/maven/plugin/surefire/AbstractSurefireMojoJava7PlusTest.java b/maven-surefire-common/src/test/java/org/apache/maven/plugin/surefire/AbstractSurefireMojoJava7PlusTest.java index 9faa4eb1f3..99f7ef03e9 100644 --- a/maven-surefire-common/src/test/java/org/apache/maven/plugin/surefire/AbstractSurefireMojoJava7PlusTest.java +++ b/maven-surefire-common/src/test/java/org/apache/maven/plugin/surefire/AbstractSurefireMojoJava7PlusTest.java @@ -589,6 +589,24 @@ public void setRunOrder( String runOrder ) } + @Override + public String getMethodRunOrder() + { + return null; + } + + @Override + public long getRandomSeed() + { + return 0; + } + + @Override + public void setRandomSeed( long seed ) + { + + } + @Override protected void handleSummary( RunResult summary, Exception firstForkException ) { diff --git a/maven-surefire-common/src/test/java/org/apache/maven/plugin/surefire/AbstractSurefireMojoTest.java b/maven-surefire-common/src/test/java/org/apache/maven/plugin/surefire/AbstractSurefireMojoTest.java index b1e7c1e3b5..bbede9b747 100644 --- a/maven-surefire-common/src/test/java/org/apache/maven/plugin/surefire/AbstractSurefireMojoTest.java +++ b/maven-surefire-common/src/test/java/org/apache/maven/plugin/surefire/AbstractSurefireMojoTest.java @@ -2078,12 +2078,30 @@ public void setRunOrder( String runOrder ) } + @Override + public String getMethodRunOrder() + { + return null; + } + @Override protected void handleSummary( RunResult summary, Exception firstForkException ) { } + @Override + public long getRandomSeed() + { + return 0; + } + + @Override + public void setRandomSeed( long seed ) + { + + } + @Override protected boolean isSkipExecution() { diff --git a/maven-surefire-common/src/test/java/org/apache/maven/plugin/surefire/MojoMocklessTest.java b/maven-surefire-common/src/test/java/org/apache/maven/plugin/surefire/MojoMocklessTest.java index ccf63f7e96..392a287631 100644 --- a/maven-surefire-common/src/test/java/org/apache/maven/plugin/surefire/MojoMocklessTest.java +++ b/maven-surefire-common/src/test/java/org/apache/maven/plugin/surefire/MojoMocklessTest.java @@ -708,6 +708,24 @@ public void setRunOrder( String runOrder ) } + @Override + public String getMethodRunOrder() + { + return null; + } + + @Override + public long getRandomSeed() + { + return 0; + } + + @Override + public void setRandomSeed( long seed ) + { + + } + @Override public String[] getDependenciesToScan() { diff --git a/maven-surefire-common/src/test/java/org/apache/maven/plugin/surefire/booterclient/BooterDeserializerProviderConfigurationTest.java b/maven-surefire-common/src/test/java/org/apache/maven/plugin/surefire/booterclient/BooterDeserializerProviderConfigurationTest.java index ca42c44faf..50b1b980dd 100644 --- a/maven-surefire-common/src/test/java/org/apache/maven/plugin/surefire/booterclient/BooterDeserializerProviderConfigurationTest.java +++ b/maven-surefire-common/src/test/java/org/apache/maven/plugin/surefire/booterclient/BooterDeserializerProviderConfigurationTest.java @@ -39,6 +39,7 @@ import org.apache.maven.surefire.testset.TestArtifactInfo; import org.apache.maven.surefire.testset.TestListResolver; import org.apache.maven.surefire.testset.TestRequest; +import org.apache.maven.surefire.util.MethodRunOrder; import org.apache.maven.surefire.util.RunOrder; import org.junit.After; import org.junit.Before; @@ -275,7 +276,8 @@ private ProviderConfiguration getTestProviderConfiguration( DirectoryScannerPara new TestRequest( getSuiteXmlFileStrings(), getTestSourceDirectory(), new TestListResolver( USER_REQUESTED_TEST + "#aUserRequestedTestMethod" ), RERUN_FAILING_TEST_COUNT ); - RunOrderParameters runOrderParameters = new RunOrderParameters( RunOrder.DEFAULT, null ); + RunOrderParameters runOrderParameters = new RunOrderParameters( RunOrder.DEFAULT, null, 0, + MethodRunOrder.DEFAULT ); return new ProviderConfiguration( directoryScannerParameters, runOrderParameters, true, reporterConfiguration, new TestArtifactInfo( "5.0", "ABC" ), testSuiteDefinition, new HashMap(), TEST_TYPED, readTestsFromInStream, cli, 0, Shutdown.DEFAULT, 0 ); diff --git a/maven-surefire-common/src/test/java/org/apache/maven/plugin/surefire/booterclient/BooterDeserializerStartupConfigurationTest.java b/maven-surefire-common/src/test/java/org/apache/maven/plugin/surefire/booterclient/BooterDeserializerStartupConfigurationTest.java index 4e9bbc9c00..f09f310890 100644 --- a/maven-surefire-common/src/test/java/org/apache/maven/plugin/surefire/booterclient/BooterDeserializerStartupConfigurationTest.java +++ b/maven-surefire-common/src/test/java/org/apache/maven/plugin/surefire/booterclient/BooterDeserializerStartupConfigurationTest.java @@ -38,6 +38,7 @@ import org.apache.maven.surefire.testset.TestArtifactInfo; import org.apache.maven.surefire.testset.TestRequest; import org.apache.maven.surefire.testset.TestListResolver; +import org.apache.maven.surefire.util.MethodRunOrder; import org.apache.maven.surefire.util.RunOrder; import org.junit.After; import org.junit.Before; @@ -196,7 +197,8 @@ private ProviderConfiguration getProviderConfiguration() new TestRequest( Arrays.asList( getSuiteXmlFileStrings() ), getTestSourceDirectory(), new TestListResolver( "aUserRequestedTest#aUserRequestedTestMethod" ) ); - RunOrderParameters runOrderParameters = new RunOrderParameters( RunOrder.DEFAULT, null ); + RunOrderParameters runOrderParameters = new RunOrderParameters( RunOrder.DEFAULT, null, 0, + MethodRunOrder.DEFAULT ); return new ProviderConfiguration( directoryScannerParameters, runOrderParameters, true, reporterConfiguration, new TestArtifactInfo( "5.0", "ABC" ), testSuiteDefinition, new HashMap(), BooterDeserializerProviderConfigurationTest.TEST_TYPED, true, cli, 0, Shutdown.DEFAULT, 0 ); diff --git a/maven-surefire-plugin/src/main/java/org/apache/maven/plugin/surefire/SurefirePlugin.java b/maven-surefire-plugin/src/main/java/org/apache/maven/plugin/surefire/SurefirePlugin.java index b39181a74f..b518de891c 100644 --- a/maven-surefire-plugin/src/main/java/org/apache/maven/plugin/surefire/SurefirePlugin.java +++ b/maven-surefire-plugin/src/main/java/org/apache/maven/plugin/surefire/SurefirePlugin.java @@ -292,6 +292,14 @@ public class SurefirePlugin @Parameter( property = "surefire.runOrder", defaultValue = "filesystem" ) private String runOrder; + + //TODO docs + @Parameter( property = "surefire.methodRunOrder", defaultValue = "default" ) + private String methodRunOrder; + + @Parameter( property = "surefire.seed" ) + private long randomSeed; + /** * A file containing include patterns. Blank lines, or lines starting with # are ignored. If {@code includes} are * also specified, these patterns are appended. Example with path, simple and regex includes: @@ -772,6 +780,18 @@ public String getRunOrder() return runOrder; } + @Override + public long getRandomSeed() + { + return randomSeed; + } + + @Override + public void setRandomSeed( long randomSeed ) + { + this.randomSeed = randomSeed; + } + @Override @SuppressWarnings( "UnusedDeclaration" ) public void setRunOrder( String runOrder ) @@ -779,6 +799,13 @@ public void setRunOrder( String runOrder ) this.runOrder = runOrder; } + + @Override + public String getMethodRunOrder() + { + return methodRunOrder; + } + @Override public File getIncludesFile() { diff --git a/surefire-api/src/main/java/org/apache/maven/surefire/booter/SurefireReflector.java b/surefire-api/src/main/java/org/apache/maven/surefire/booter/SurefireReflector.java index 5cc1415e9a..2f1e2b27ff 100644 --- a/surefire-api/src/main/java/org/apache/maven/surefire/booter/SurefireReflector.java +++ b/surefire-api/src/main/java/org/apache/maven/surefire/booter/SurefireReflector.java @@ -204,10 +204,11 @@ private Object createRunOrderParameters( RunOrderParameters runOrderParameters ) return null; } //Can't use the constructor with the RunOrder parameter. Using it causes some integration tests to fail. - Class[] arguments = { String.class, File.class }; + Class[] arguments = { String.class, File.class, Long.TYPE, String.class }; Constructor constructor = getConstructor( this.runOrderParameters, arguments ); File runStatisticsFile = runOrderParameters.getRunStatisticsFile(); - return newInstance( constructor, RunOrder.asString( runOrderParameters.getRunOrder() ), runStatisticsFile ); + return newInstance( constructor, RunOrder.asString( runOrderParameters.getRunOrder() ), runStatisticsFile, + runOrderParameters.getRandomSeed(), runOrderParameters.getMethodRunOrder().toString() ); } private Object createTestArtifactInfo( TestArtifactInfo testArtifactInfo ) diff --git a/surefire-api/src/main/java/org/apache/maven/surefire/testset/RunOrderParameters.java b/surefire-api/src/main/java/org/apache/maven/surefire/testset/RunOrderParameters.java index 6a2cceb746..8e1813e0b0 100644 --- a/surefire-api/src/main/java/org/apache/maven/surefire/testset/RunOrderParameters.java +++ b/surefire-api/src/main/java/org/apache/maven/surefire/testset/RunOrderParameters.java @@ -19,9 +19,11 @@ * under the License. */ -import java.io.File; +import org.apache.maven.surefire.util.MethodRunOrder; import org.apache.maven.surefire.util.RunOrder; +import java.io.File; + /** * @author Kristian Rosenvold */ @@ -31,21 +33,30 @@ public class RunOrderParameters private File runStatisticsFile; - public RunOrderParameters( RunOrder[] runOrder, File runStatisticsFile ) + private final MethodRunOrder methodRunOrder; + + private final long randomSeed; + + public RunOrderParameters( RunOrder[] runOrder, File runStatisticsFile, long randomSeed, + MethodRunOrder methodRunOrder ) { this.runOrder = runOrder; this.runStatisticsFile = runStatisticsFile; + this.methodRunOrder = methodRunOrder; + this.randomSeed = randomSeed; } - public RunOrderParameters( String runOrder, File runStatisticsFile ) + public RunOrderParameters( String runOrder, File runStatisticsFile, long randomSeed, String methodRunOrder ) { this.runOrder = runOrder == null ? RunOrder.DEFAULT : RunOrder.valueOfMulti( runOrder ); this.runStatisticsFile = runStatisticsFile; + this.methodRunOrder = MethodRunOrder.valueOf( methodRunOrder ); + this.randomSeed = randomSeed; } public static RunOrderParameters alphabetical() { - return new RunOrderParameters( new RunOrder[]{ RunOrder.ALPHABETICAL }, null ); + return new RunOrderParameters( new RunOrder[]{ RunOrder.ALPHABETICAL }, null, 0, MethodRunOrder.DEFAULT ); } public RunOrder[] getRunOrder() @@ -58,4 +69,14 @@ public File getRunStatisticsFile() return runStatisticsFile; } + public MethodRunOrder getMethodRunOrder() + { + return methodRunOrder; + } + + public long getRandomSeed() + { + return randomSeed; + } + } diff --git a/surefire-api/src/main/java/org/apache/maven/surefire/util/DefaultRunOrderCalculator.java b/surefire-api/src/main/java/org/apache/maven/surefire/util/DefaultRunOrderCalculator.java index 22a219d25a..776352e31e 100644 --- a/surefire-api/src/main/java/org/apache/maven/surefire/util/DefaultRunOrderCalculator.java +++ b/surefire-api/src/main/java/org/apache/maven/surefire/util/DefaultRunOrderCalculator.java @@ -26,8 +26,10 @@ import java.util.Calendar; import java.util.Collections; import java.util.Comparator; +import java.util.HashMap; import java.util.LinkedHashSet; import java.util.List; +import java.util.Random; /** * Applies the final runorder of the tests @@ -45,12 +47,16 @@ public class DefaultRunOrderCalculator private final int threadCount; + private final Random random; + + public DefaultRunOrderCalculator( RunOrderParameters runOrderParameters, int threadCount ) { this.runOrderParameters = runOrderParameters; this.threadCount = threadCount; this.runOrder = runOrderParameters.getRunOrder(); this.sortOrder = this.runOrder.length > 0 ? getSortOrderComparator( this.runOrder[0] ) : null; + this.random = new Random( runOrderParameters.getRandomSeed() ); } @Override @@ -68,11 +74,75 @@ public TestsToRun orderTestClasses( TestsToRun scannedClasses ) return new TestsToRun( new LinkedHashSet<>( result ) ); } + @Override + public Comparator comparatorForTestMethods() + { + MethodRunOrder order = runOrderParameters.getMethodRunOrder(); + if ( MethodRunOrder.DEFAULT.equals( order ) ) + { + return null; + } + else if ( MethodRunOrder.ALPHABETICAL.equals( order ) ) + { + return new Comparator() + { + @Override + public int compare( String o1, String o2 ) + { + return o1.compareTo( o2 ); + } + }; + } + else if ( MethodRunOrder.REVERSE_ALPHABETICAL.equals( order ) ) + { + return new Comparator() + { + @Override + public int compare( String o1, String o2 ) + { + return o2.compareTo( o1 ); + } + }; + } + else if ( MethodRunOrder.RANDOM.equals( order ) ) + { + return new Comparator() + { + HashMap randomVals = new HashMap<>(); + + private int getRandom( String obj ) + { + if ( !randomVals.containsKey( obj ) ) + { + randomVals.put( obj, random.nextInt() ); + } + return randomVals.get( obj ); + } + + @Override + public int compare( String o1, String o2 ) + { + int i1 = getRandom( o1 ); + int i2 = getRandom( o2 ); + return ( i1 > i2 ? 1 : -1 ); + } + }; + } + else if ( MethodRunOrder.FLAKY_FINDING.equals( order ) ) + { + throw new UnsupportedOperationException( "Unimplemented" ); + } + else + { + throw new UnsupportedOperationException( "Unsupported method run order: " + order.name() ); + } + } + private void orderTestClasses( List> testClasses, RunOrder runOrder ) { if ( RunOrder.RANDOM.equals( runOrder ) ) { - Collections.shuffle( testClasses ); + Collections.shuffle( testClasses, random ); } else if ( RunOrder.FAILEDFIRST.equals( runOrder ) ) { diff --git a/surefire-api/src/main/java/org/apache/maven/surefire/util/MethodRunOrder.java b/surefire-api/src/main/java/org/apache/maven/surefire/util/MethodRunOrder.java new file mode 100644 index 0000000000..8d2d71e7a1 --- /dev/null +++ b/surefire-api/src/main/java/org/apache/maven/surefire/util/MethodRunOrder.java @@ -0,0 +1,100 @@ +package org.apache.maven.surefire.util; + +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +/** + * TODO docs + */ +public class MethodRunOrder +{ + + public static final MethodRunOrder ALPHABETICAL = new MethodRunOrder( "alphabetical" ); + public static final MethodRunOrder REVERSE_ALPHABETICAL = new MethodRunOrder( "reversealphabetical" ); + public static final MethodRunOrder RANDOM = new MethodRunOrder( "random" ); + public static final MethodRunOrder FLAKY_FINDING = new MethodRunOrder( "flakyfinding" ); + public static final MethodRunOrder DEFAULT = new MethodRunOrder( "default" ); + + private final String name; + + MethodRunOrder( String name ) + { + this.name = name; + } + + private static MethodRunOrder[] values() + { + return new MethodRunOrder[] {ALPHABETICAL, REVERSE_ALPHABETICAL, RANDOM, FLAKY_FINDING, DEFAULT}; + } + + public boolean matches( String anotherName ) + { + return name.equalsIgnoreCase( anotherName ); + } + + public static MethodRunOrder valueOf( String name ) + { + if ( name == null ) + { + return null; + } + else + { + for ( MethodRunOrder each : values() ) + { + if ( each.matches( name ) ) + { + return each; + } + } + } + String errorMessage = createMessageForMissingRunOrder( name ); + throw new IllegalArgumentException( errorMessage ); + + } + + private static String createMessageForMissingRunOrder( String name ) + { + MethodRunOrder[] methodRunOrders = values(); + StringBuilder message = new StringBuilder( "There's no MethodRunOrder with the name " ); + message.append( name ); + message.append( ". Please use one of the following RunOrders: " ); + for ( int i = 0; i < methodRunOrders.length; i++ ) + { + if ( i != 0 ) + { + message.append( ", " ); + } + message.append( methodRunOrders[i] ); + } + message.append( '.' ); + return message.toString(); + } + + @Override + public String toString() + { + return name; + } + + public String name() + { + return name; + } +} diff --git a/surefire-api/src/main/java/org/apache/maven/surefire/util/RunOrderCalculator.java b/surefire-api/src/main/java/org/apache/maven/surefire/util/RunOrderCalculator.java index bdbc48b790..863790ac60 100644 --- a/surefire-api/src/main/java/org/apache/maven/surefire/util/RunOrderCalculator.java +++ b/surefire-api/src/main/java/org/apache/maven/surefire/util/RunOrderCalculator.java @@ -19,10 +19,14 @@ * under the License. */ +import java.util.Comparator; + /** * @author Kristian Rosenvold */ public interface RunOrderCalculator { TestsToRun orderTestClasses( TestsToRun scannedClasses ); + + Comparator comparatorForTestMethods(); } diff --git a/surefire-api/src/test/java/org/apache/maven/surefire/booter/SurefireReflectorTest.java b/surefire-api/src/test/java/org/apache/maven/surefire/booter/SurefireReflectorTest.java index 0a006bfd24..8fce527d94 100644 --- a/surefire-api/src/test/java/org/apache/maven/surefire/booter/SurefireReflectorTest.java +++ b/surefire-api/src/test/java/org/apache/maven/surefire/booter/SurefireReflectorTest.java @@ -30,6 +30,7 @@ import org.apache.maven.surefire.testset.TestArtifactInfo; import org.apache.maven.surefire.testset.TestListResolver; import org.apache.maven.surefire.testset.TestRequest; +import org.apache.maven.surefire.util.MethodRunOrder; import org.apache.maven.surefire.util.RunOrder; import org.mockito.ArgumentCaptor; @@ -112,7 +113,8 @@ public void testRunOrderParameters() SurefireReflector surefireReflector = getReflector(); Object foo = getFoo(); - RunOrderParameters runOrderParameters = new RunOrderParameters( RunOrder.DEFAULT, new File( "." ) ); + RunOrderParameters runOrderParameters = new RunOrderParameters( RunOrder.DEFAULT, new File( "." ), 0L, + MethodRunOrder.DEFAULT ); surefireReflector.setRunOrderParameters( foo, runOrderParameters ); assertTrue( isCalled( foo ) ); } diff --git a/surefire-booter/src/main/java/org/apache/maven/surefire/booter/BooterConstants.java b/surefire-booter/src/main/java/org/apache/maven/surefire/booter/BooterConstants.java index fc570f0fa0..8b8422c197 100644 --- a/surefire-booter/src/main/java/org/apache/maven/surefire/booter/BooterConstants.java +++ b/surefire-booter/src/main/java/org/apache/maven/surefire/booter/BooterConstants.java @@ -47,6 +47,8 @@ private BooterConstants() public static final String TEST_CLASSES_DIRECTORY = "testClassesDirectory"; public static final String RUN_ORDER = "runOrder"; public static final String RUN_STATISTICS_FILE = "runStatisticsFile"; + public static final String METHOD_RUN_ORDER = "methodRunOrder"; + public static final String RANDOM_SEED = "randomSeed"; public static final String TEST_SUITE_XML_FILES = "testSuiteXmlFiles"; public static final String PROVIDER_CONFIGURATION = "providerConfiguration"; public static final String FORKTESTSET = "forkTestSet"; diff --git a/surefire-booter/src/main/java/org/apache/maven/surefire/booter/BooterDeserializer.java b/surefire-booter/src/main/java/org/apache/maven/surefire/booter/BooterDeserializer.java index b0916795c6..177ed3f3ba 100644 --- a/surefire-booter/src/main/java/org/apache/maven/surefire/booter/BooterDeserializer.java +++ b/surefire-booter/src/main/java/org/apache/maven/surefire/booter/BooterDeserializer.java @@ -87,6 +87,8 @@ public ProviderConfiguration deserialize() final File testClassesDirectory = properties.getFileProperty( TEST_CLASSES_DIRECTORY ); final String runOrder = properties.getProperty( RUN_ORDER ); final String runStatisticsFile = properties.getProperty( RUN_STATISTICS_FILE ); + final String methodRunOrder = properties.getProperty( METHOD_RUN_ORDER ); + final long randomSeed = properties.getLongProperty( RANDOM_SEED ); final int rerunFailingTestsCount = properties.getIntProperty( RERUN_FAILING_TESTS_COUNT ); @@ -95,7 +97,8 @@ public ProviderConfiguration deserialize() properties.getBooleanProperty( FAILIFNOTESTS ), runOrder ); RunOrderParameters runOrderParameters - = new RunOrderParameters( runOrder, runStatisticsFile == null ? null : new File( runStatisticsFile ) ); + = new RunOrderParameters( runOrder, runStatisticsFile == null ? null : new File( runStatisticsFile ), + randomSeed, methodRunOrder ); TestArtifactInfo testNg = new TestArtifactInfo( testNgVersion, testArtifactClassifier ); TestRequest testSuiteDefinition = diff --git a/surefire-providers/surefire-junit4/src/main/java/org/apache/maven/surefire/junit4/JUnit4Provider.java b/surefire-providers/surefire-junit4/src/main/java/org/apache/maven/surefire/junit4/JUnit4Provider.java index b9649331ae..9795e275e7 100644 --- a/surefire-providers/surefire-junit4/src/main/java/org/apache/maven/surefire/junit4/JUnit4Provider.java +++ b/surefire-providers/surefire-junit4/src/main/java/org/apache/maven/surefire/junit4/JUnit4Provider.java @@ -48,6 +48,7 @@ import org.junit.runner.notification.StoppedByUserException; import java.util.Collection; +import java.util.Comparator; import java.util.Set; import static java.lang.reflect.Modifier.isAbstract; @@ -270,7 +271,8 @@ private void executeWithRerun( Class clazz, Notifier notifier ) try { notifier.asFailFast( isFailFast() ); - execute( clazz, notifier, hasMethodFilter ? createMethodFilter() : null ); + execute( clazz, notifier, hasMethodFilter ? createMethodFilter() : null, + runOrderCalculator.comparatorForTestMethods() ); } finally { @@ -287,7 +289,8 @@ private void executeWithRerun( Class clazz, Notifier notifier ) Set failures = generateFailingTestDescriptions( failureListener.getAllFailures() ); failureListener.reset(); Filter failureDescriptionFilter = createMatchAnyDescriptionFilter( failures ); - execute( clazz, rerunNotifier, failureDescriptionFilter ); + execute( clazz, rerunNotifier, failureDescriptionFilter, + runOrderCalculator.comparatorForTestMethods() ); } } } @@ -349,12 +352,24 @@ private static boolean isJUnit4UpgradeCheck() return System.getProperty( "surefire.junit4.upgradecheck" ) != null; } - private static void execute( Class testClass, Notifier notifier, Filter filter ) + private static void execute( Class testClass, Notifier notifier, Filter filter, + final Comparator runOrderComparator ) { final int classModifiers = testClass.getModifiers(); if ( !isAbstract( classModifiers ) && !isInterface( classModifiers ) ) { Request request = aClass( testClass ); + if ( runOrderComparator != null ) + { + request = request.sortWith( new Comparator() + { + @Override + public int compare( Description o1, Description o2 ) + { + return runOrderComparator.compare( o1.toString(), o2.toString() ); + } + } ); + } if ( filter != null ) { request = request.filterWith( filter ); diff --git a/surefire-providers/surefire-junit47/src/main/java/org/apache/maven/surefire/junitcore/JUnitCoreProvider.java b/surefire-providers/surefire-junit47/src/main/java/org/apache/maven/surefire/junitcore/JUnitCoreProvider.java index 7c74e8b866..d3747666e6 100644 --- a/surefire-providers/surefire-junit47/src/main/java/org/apache/maven/surefire/junitcore/JUnitCoreProvider.java +++ b/surefire-providers/surefire-junit47/src/main/java/org/apache/maven/surefire/junitcore/JUnitCoreProvider.java @@ -146,7 +146,8 @@ public RunResult invoke( Object forkTestSet ) try { - JUnitCoreWrapper core = new JUnitCoreWrapper( notifier, jUnitCoreParameters, consoleStream ); + JUnitCoreWrapper core = new JUnitCoreWrapper( notifier, jUnitCoreParameters, consoleStream, + runOrderCalculator ); if ( commandsReader != null ) { @@ -163,7 +164,8 @@ public RunResult invoke( Object forkTestSet ) { Notifier rerunNotifier = pureNotifier(); notifier.copyListenersTo( rerunNotifier ); - JUnitCoreWrapper rerunCore = new JUnitCoreWrapper( rerunNotifier, jUnitCoreParameters, consoleStream ); + JUnitCoreWrapper rerunCore = new JUnitCoreWrapper( rerunNotifier, jUnitCoreParameters, consoleStream, + runOrderCalculator ); for ( int i = 0; i < rerunFailingTestsCount && !testFailureListener.getAllFailures().isEmpty(); i++ ) { Set failures = generateFailingTestDescriptions( testFailureListener.getAllFailures() ); diff --git a/surefire-providers/surefire-junit47/src/main/java/org/apache/maven/surefire/junitcore/JUnitCoreWrapper.java b/surefire-providers/surefire-junit47/src/main/java/org/apache/maven/surefire/junitcore/JUnitCoreWrapper.java index 656d55bcfb..2b0e07b279 100644 --- a/surefire-providers/surefire-junit47/src/main/java/org/apache/maven/surefire/junitcore/JUnitCoreWrapper.java +++ b/surefire-providers/surefire-junit47/src/main/java/org/apache/maven/surefire/junitcore/JUnitCoreWrapper.java @@ -24,6 +24,7 @@ import org.apache.maven.surefire.junitcore.pc.ParallelComputerBuilder; import org.apache.maven.surefire.report.ConsoleStream; import org.apache.maven.surefire.testset.TestSetFailedException; +import org.apache.maven.surefire.util.RunOrderCalculator; import org.apache.maven.surefire.util.TestsToRun; import org.junit.Ignore; import org.junit.runner.Computer; @@ -36,6 +37,7 @@ import java.util.Collection; import java.util.Collections; +import java.util.Comparator; import java.util.Iterator; import java.util.Queue; @@ -55,12 +57,15 @@ final class JUnitCoreWrapper private final Notifier notifier; private final JUnitCoreParameters jUnitCoreParameters; private final ConsoleStream consoleStream; + private final RunOrderCalculator runOrderCalculator; - JUnitCoreWrapper( Notifier notifier, JUnitCoreParameters jUnitCoreParameters, ConsoleStream consoleStream ) + JUnitCoreWrapper( Notifier notifier, JUnitCoreParameters jUnitCoreParameters, ConsoleStream consoleStream, + RunOrderCalculator runOrderCalculator ) { this.notifier = notifier; this.jUnitCoreParameters = jUnitCoreParameters; this.consoleStream = consoleStream; + this.runOrderCalculator = runOrderCalculator; } void execute( TestsToRun testsToRun, Filter filter ) @@ -124,6 +129,18 @@ private void createRequestAndRun( Filter filter, Computer computer, JUnitCore ju throws TestSetFailedException { Request req = classes( computer, classesToRun ); + final Comparator testOrderComparator = runOrderCalculator.comparatorForTestMethods(); + if ( testOrderComparator != null ) + { + req = req.sortWith( new Comparator() + { + @Override + public int compare( Description o1, Description o2 ) + { + return testOrderComparator.compare( o1.toString(), o2.toString() ); + } + } ); + } if ( filter != null ) { req = new FilteringRequest( req, filter ); diff --git a/surefire-providers/surefire-junit47/src/test/java/org/apache/maven/surefire/junitcore/Surefire746Test.java b/surefire-providers/surefire-junit47/src/test/java/org/apache/maven/surefire/junitcore/Surefire746Test.java index 8859c195fb..7a530cffaf 100644 --- a/surefire-providers/surefire-junit47/src/test/java/org/apache/maven/surefire/junitcore/Surefire746Test.java +++ b/surefire-providers/surefire-junit47/src/test/java/org/apache/maven/surefire/junitcore/Surefire746Test.java @@ -54,7 +54,9 @@ import org.apache.maven.surefire.report.ReporterFactory; import org.apache.maven.surefire.report.RunListener; import org.apache.maven.surefire.suite.RunResult; +import org.apache.maven.surefire.testset.RunOrderParameters; import org.apache.maven.surefire.testset.TestSetFailedException; +import org.apache.maven.surefire.util.DefaultRunOrderCalculator; import org.apache.maven.surefire.util.TestsToRun; import org.junit.Rule; @@ -137,7 +139,9 @@ public void surefireIsConfused_ByMultipleIgnore_OnClassLevel() throws Exception exception.expect( TestSetFailedException.class ); JUnit4RunListener dummy = new JUnit4RunListener( new MockReporter() ); new JUnitCoreWrapper( new Notifier( dummy, 0 ), jUnitCoreParameters, - new DefaultDirectConsoleReporter( System.out ) ).execute( testsToRun, customRunListeners, null ); + new DefaultDirectConsoleReporter( System.out ), + new DefaultRunOrderCalculator( RunOrderParameters.alphabetical(), 1 ) ). + execute( testsToRun, customRunListeners, null ); } finally { From 7da3a987dddcda33d8f3e546100de83490dfff0d Mon Sep 17 00:00:00 2001 From: Jonathan Bell Date: Mon, 13 Apr 2020 15:43:39 -0400 Subject: [PATCH 03/24] also add a bunch of hacks that look less pretty --- .../util/DefaultRunOrderCalculator.java | 75 ++++++++++++++++++- 1 file changed, 73 insertions(+), 2 deletions(-) diff --git a/surefire-api/src/main/java/org/apache/maven/surefire/util/DefaultRunOrderCalculator.java b/surefire-api/src/main/java/org/apache/maven/surefire/util/DefaultRunOrderCalculator.java index 776352e31e..af15a93fc3 100644 --- a/surefire-api/src/main/java/org/apache/maven/surefire/util/DefaultRunOrderCalculator.java +++ b/surefire-api/src/main/java/org/apache/maven/surefire/util/DefaultRunOrderCalculator.java @@ -28,6 +28,7 @@ import java.util.Comparator; import java.util.HashMap; import java.util.LinkedHashSet; +import java.util.LinkedList; import java.util.List; import java.util.Random; @@ -130,7 +131,51 @@ public int compare( String o1, String o2 ) } else if ( MethodRunOrder.FLAKY_FINDING.equals( order ) ) { - throw new UnsupportedOperationException( "Unimplemented" ); + String orderParam = System.getProperty( "flakyTestOrder" ); + if ( orderParam == null ) + { + throw new IllegalStateException( "Please set system property flakyTestOrder to use flaky finding" ); + } + final HashMap orders = new HashMap<>(); + int i = 0; + for ( String s : orderParam.split( "," ) ) + { + orders.put( s, i ); + i++; + if ( i > 2 ) + { + throw new UnsupportedOperationException( "This only supports 2 tests at a time for now." ); + } + } + return new Comparator() + { + int getRank( String o ) + { + synchronized ( orders ) + { + if ( !orders.containsKey( o ) ) + { + orders.put( o, orders.size() ); + } + return orders.get( o ); + } + } + + @Override + public int compare( String o1, String o2 ) + { + int r1 = getRank( o1 ); + int r2 = getRank( o2 ); + if ( r1 < r2 ) + { + return -1; + } + else + { + return 1; + } + } + }; } else { @@ -140,7 +185,13 @@ else if ( MethodRunOrder.FLAKY_FINDING.equals( order ) ) private void orderTestClasses( List> testClasses, RunOrder runOrder ) { - if ( RunOrder.RANDOM.equals( runOrder ) ) + if ( System.getProperty( "flakyTestOrder" ) != null ) + { + List> sorted = sortClassesBySpecifiedOrder( testClasses, System.getProperty( "flakyTestOrder" ) ); + testClasses.clear(); + testClasses.addAll( sorted ); + } + else if ( RunOrder.RANDOM.equals( runOrder ) ) { Collections.shuffle( testClasses, random ); } @@ -166,6 +217,26 @@ else if ( sortOrder != null ) } } + private List> sortClassesBySpecifiedOrder( List> testClasses, String flakyTestOrder ) + { + HashMap> classes = new HashMap<>(); + for ( Class each : testClasses ) + { + classes.put( each.getName(), each ); + } + LinkedList> ret = new LinkedList<>(); + for ( String s : flakyTestOrder.split( "," ) ) + { + String testClass = s.substring( s.indexOf( '(' ) + 1, s.length() - 1 ); + Class c = classes.remove( testClass ); + if ( c != null ) + { + ret.add( c ); + } + } + return ret; + } + private Comparator getSortOrderComparator( RunOrder runOrder ) { if ( RunOrder.ALPHABETICAL.equals( runOrder ) ) From b062f9fe948f54bfab7ecb80246a789d829b695a Mon Sep 17 00:00:00 2001 From: Jonathan Bell Date: Tue, 14 Apr 2020 09:21:41 -0400 Subject: [PATCH 04/24] Bump version --- maven-failsafe-plugin/pom.xml | 2 +- maven-surefire-common/pom.xml | 2 +- maven-surefire-plugin/pom.xml | 2 +- maven-surefire-report-plugin/pom.xml | 2 +- pom.xml | 2 +- surefire-api/pom.xml | 2 +- surefire-booter/pom.xml | 2 +- surefire-extensions-api/pom.xml | 2 +- surefire-grouper/pom.xml | 2 +- surefire-its/pom.xml | 2 +- surefire-logger-api/pom.xml | 2 +- surefire-providers/common-java5/pom.xml | 2 +- surefire-providers/common-junit3/pom.xml | 2 +- surefire-providers/common-junit4/pom.xml | 2 +- surefire-providers/common-junit48/pom.xml | 2 +- surefire-providers/pom.xml | 2 +- surefire-providers/surefire-junit-platform/pom.xml | 2 +- surefire-providers/surefire-junit3/pom.xml | 2 +- surefire-providers/surefire-junit4/pom.xml | 2 +- surefire-providers/surefire-junit47/pom.xml | 2 +- surefire-providers/surefire-testng-utils/pom.xml | 2 +- surefire-providers/surefire-testng/pom.xml | 2 +- surefire-report-parser/pom.xml | 2 +- surefire-shadefire/pom.xml | 2 +- surefire-shared-utils/pom.xml | 2 +- 25 files changed, 25 insertions(+), 25 deletions(-) diff --git a/maven-failsafe-plugin/pom.xml b/maven-failsafe-plugin/pom.xml index a6474f4790..de6885bdb4 100644 --- a/maven-failsafe-plugin/pom.xml +++ b/maven-failsafe-plugin/pom.xml @@ -24,7 +24,7 @@ org.apache.maven.surefire surefire - 3.0.0-SNAPSHOT + 3.0.0-FLAKYSNAPSHOT org.apache.maven.plugins diff --git a/maven-surefire-common/pom.xml b/maven-surefire-common/pom.xml index fefe8319b8..6bea5e1f8c 100644 --- a/maven-surefire-common/pom.xml +++ b/maven-surefire-common/pom.xml @@ -24,7 +24,7 @@ org.apache.maven.surefire surefire - 3.0.0-SNAPSHOT + 3.0.0-FLAKYSNAPSHOT maven-surefire-common diff --git a/maven-surefire-plugin/pom.xml b/maven-surefire-plugin/pom.xml index 4901792a9b..dc3c99f909 100644 --- a/maven-surefire-plugin/pom.xml +++ b/maven-surefire-plugin/pom.xml @@ -24,7 +24,7 @@ org.apache.maven.surefire surefire - 3.0.0-SNAPSHOT + 3.0.0-FLAKYSNAPSHOT org.apache.maven.plugins diff --git a/maven-surefire-report-plugin/pom.xml b/maven-surefire-report-plugin/pom.xml index 48708fe75c..15e224e9aa 100644 --- a/maven-surefire-report-plugin/pom.xml +++ b/maven-surefire-report-plugin/pom.xml @@ -24,7 +24,7 @@ org.apache.maven.surefire surefire - 3.0.0-SNAPSHOT + 3.0.0-FLAKYSNAPSHOT org.apache.maven.plugins diff --git a/pom.xml b/pom.xml index f4f992ef10..168dd0ee75 100644 --- a/pom.xml +++ b/pom.xml @@ -28,7 +28,7 @@ org.apache.maven.surefire surefire - 3.0.0-SNAPSHOT + 3.0.0-FLAKYSNAPSHOT pom Apache Maven Surefire diff --git a/surefire-api/pom.xml b/surefire-api/pom.xml index 7a91275b0b..8fed381743 100644 --- a/surefire-api/pom.xml +++ b/surefire-api/pom.xml @@ -23,7 +23,7 @@ org.apache.maven.surefire surefire - 3.0.0-SNAPSHOT + 3.0.0-FLAKYSNAPSHOT surefire-api diff --git a/surefire-booter/pom.xml b/surefire-booter/pom.xml index 1fa5045f30..8b7a6398d2 100644 --- a/surefire-booter/pom.xml +++ b/surefire-booter/pom.xml @@ -23,7 +23,7 @@ org.apache.maven.surefire surefire - 3.0.0-SNAPSHOT + 3.0.0-FLAKYSNAPSHOT surefire-booter diff --git a/surefire-extensions-api/pom.xml b/surefire-extensions-api/pom.xml index 1e88403a0c..57408ea2f3 100644 --- a/surefire-extensions-api/pom.xml +++ b/surefire-extensions-api/pom.xml @@ -24,7 +24,7 @@ org.apache.maven.surefire surefire - 3.0.0-SNAPSHOT + 3.0.0-FLAKYSNAPSHOT surefire-extensions-api diff --git a/surefire-grouper/pom.xml b/surefire-grouper/pom.xml index 2da31a0980..49298d1704 100644 --- a/surefire-grouper/pom.xml +++ b/surefire-grouper/pom.xml @@ -24,7 +24,7 @@ org.apache.maven.surefire surefire - 3.0.0-SNAPSHOT + 3.0.0-FLAKYSNAPSHOT surefire-grouper diff --git a/surefire-its/pom.xml b/surefire-its/pom.xml index a49244f8ad..b64a645b4c 100644 --- a/surefire-its/pom.xml +++ b/surefire-its/pom.xml @@ -24,7 +24,7 @@ org.apache.maven.surefire surefire - 3.0.0-SNAPSHOT + 3.0.0-FLAKYSNAPSHOT surefire-its diff --git a/surefire-logger-api/pom.xml b/surefire-logger-api/pom.xml index bcac4ad2f3..3f853589ed 100644 --- a/surefire-logger-api/pom.xml +++ b/surefire-logger-api/pom.xml @@ -23,7 +23,7 @@ org.apache.maven.surefire surefire - 3.0.0-SNAPSHOT + 3.0.0-FLAKYSNAPSHOT surefire-logger-api diff --git a/surefire-providers/common-java5/pom.xml b/surefire-providers/common-java5/pom.xml index 9245b977b4..20e21e75b0 100644 --- a/surefire-providers/common-java5/pom.xml +++ b/surefire-providers/common-java5/pom.xml @@ -23,7 +23,7 @@ org.apache.maven.surefire surefire-providers - 3.0.0-SNAPSHOT + 3.0.0-FLAKYSNAPSHOT common-java5 diff --git a/surefire-providers/common-junit3/pom.xml b/surefire-providers/common-junit3/pom.xml index 67c2d35c12..49f28aa3a3 100644 --- a/surefire-providers/common-junit3/pom.xml +++ b/surefire-providers/common-junit3/pom.xml @@ -23,7 +23,7 @@ org.apache.maven.surefire surefire-providers - 3.0.0-SNAPSHOT + 3.0.0-FLAKYSNAPSHOT common-junit3 diff --git a/surefire-providers/common-junit4/pom.xml b/surefire-providers/common-junit4/pom.xml index 78c8b0ca71..0a32bd5284 100644 --- a/surefire-providers/common-junit4/pom.xml +++ b/surefire-providers/common-junit4/pom.xml @@ -23,7 +23,7 @@ org.apache.maven.surefire surefire-providers - 3.0.0-SNAPSHOT + 3.0.0-FLAKYSNAPSHOT common-junit4 diff --git a/surefire-providers/common-junit48/pom.xml b/surefire-providers/common-junit48/pom.xml index fa80e47cee..d61ad8526e 100644 --- a/surefire-providers/common-junit48/pom.xml +++ b/surefire-providers/common-junit48/pom.xml @@ -23,7 +23,7 @@ org.apache.maven.surefire surefire-providers - 3.0.0-SNAPSHOT + 3.0.0-FLAKYSNAPSHOT common-junit48 diff --git a/surefire-providers/pom.xml b/surefire-providers/pom.xml index 2b83b4f5bb..51c99bb9b4 100644 --- a/surefire-providers/pom.xml +++ b/surefire-providers/pom.xml @@ -24,7 +24,7 @@ org.apache.maven.surefire surefire - 3.0.0-SNAPSHOT + 3.0.0-FLAKYSNAPSHOT surefire-providers diff --git a/surefire-providers/surefire-junit-platform/pom.xml b/surefire-providers/surefire-junit-platform/pom.xml index 82185572b5..4e5d098b83 100644 --- a/surefire-providers/surefire-junit-platform/pom.xml +++ b/surefire-providers/surefire-junit-platform/pom.xml @@ -23,7 +23,7 @@ org.apache.maven.surefire surefire-providers - 3.0.0-SNAPSHOT + 3.0.0-FLAKYSNAPSHOT surefire-junit-platform diff --git a/surefire-providers/surefire-junit3/pom.xml b/surefire-providers/surefire-junit3/pom.xml index 1bd170ad88..1733a0cb77 100644 --- a/surefire-providers/surefire-junit3/pom.xml +++ b/surefire-providers/surefire-junit3/pom.xml @@ -23,7 +23,7 @@ org.apache.maven.surefire surefire-providers - 3.0.0-SNAPSHOT + 3.0.0-FLAKYSNAPSHOT surefire-junit3 diff --git a/surefire-providers/surefire-junit4/pom.xml b/surefire-providers/surefire-junit4/pom.xml index e47926e252..d6af62d3e4 100644 --- a/surefire-providers/surefire-junit4/pom.xml +++ b/surefire-providers/surefire-junit4/pom.xml @@ -23,7 +23,7 @@ org.apache.maven.surefire surefire-providers - 3.0.0-SNAPSHOT + 3.0.0-FLAKYSNAPSHOT surefire-junit4 diff --git a/surefire-providers/surefire-junit47/pom.xml b/surefire-providers/surefire-junit47/pom.xml index 05ddfa4fe5..98846d6035 100644 --- a/surefire-providers/surefire-junit47/pom.xml +++ b/surefire-providers/surefire-junit47/pom.xml @@ -23,7 +23,7 @@ org.apache.maven.surefire surefire-providers - 3.0.0-SNAPSHOT + 3.0.0-FLAKYSNAPSHOT surefire-junit47 diff --git a/surefire-providers/surefire-testng-utils/pom.xml b/surefire-providers/surefire-testng-utils/pom.xml index 7867b4d37a..e815faef83 100644 --- a/surefire-providers/surefire-testng-utils/pom.xml +++ b/surefire-providers/surefire-testng-utils/pom.xml @@ -23,7 +23,7 @@ org.apache.maven.surefire surefire-providers - 3.0.0-SNAPSHOT + 3.0.0-FLAKYSNAPSHOT surefire-testng-utils diff --git a/surefire-providers/surefire-testng/pom.xml b/surefire-providers/surefire-testng/pom.xml index 2dddddc8eb..11c55a3fa4 100644 --- a/surefire-providers/surefire-testng/pom.xml +++ b/surefire-providers/surefire-testng/pom.xml @@ -23,7 +23,7 @@ org.apache.maven.surefire surefire-providers - 3.0.0-SNAPSHOT + 3.0.0-FLAKYSNAPSHOT surefire-testng diff --git a/surefire-report-parser/pom.xml b/surefire-report-parser/pom.xml index b9866c3ca9..f1ce6b3ee9 100644 --- a/surefire-report-parser/pom.xml +++ b/surefire-report-parser/pom.xml @@ -24,7 +24,7 @@ org.apache.maven.surefire surefire - 3.0.0-SNAPSHOT + 3.0.0-FLAKYSNAPSHOT surefire-report-parser diff --git a/surefire-shadefire/pom.xml b/surefire-shadefire/pom.xml index 6ade0321cf..34e126619c 100644 --- a/surefire-shadefire/pom.xml +++ b/surefire-shadefire/pom.xml @@ -23,7 +23,7 @@ org.apache.maven.surefire surefire - 3.0.0-SNAPSHOT + 3.0.0-FLAKYSNAPSHOT surefire-shadefire diff --git a/surefire-shared-utils/pom.xml b/surefire-shared-utils/pom.xml index 3e3edef9da..48989bee3b 100644 --- a/surefire-shared-utils/pom.xml +++ b/surefire-shared-utils/pom.xml @@ -24,7 +24,7 @@ org.apache.maven.surefire surefire - 3.0.0-SNAPSHOT + 3.0.0-FLAKYSNAPSHOT surefire-shared-utils From d0163cf26e5465fb6d0bafab4450e1476dc38ef3 Mon Sep 17 00:00:00 2001 From: Wing Lam Date: Tue, 21 Jul 2020 03:24:09 +0000 Subject: [PATCH 05/24] Updating plugin. --- .../maven/plugin/surefire/SurefirePlugin.java | 31 +++++++++++++++++++ pom.xml | 2 +- .../util/DefaultRunOrderCalculator.java | 14 ++++----- 3 files changed, 39 insertions(+), 8 deletions(-) diff --git a/maven-surefire-plugin/src/main/java/org/apache/maven/plugin/surefire/SurefirePlugin.java b/maven-surefire-plugin/src/main/java/org/apache/maven/plugin/surefire/SurefirePlugin.java index b518de891c..fd2e1a2a87 100644 --- a/maven-surefire-plugin/src/main/java/org/apache/maven/plugin/surefire/SurefirePlugin.java +++ b/maven-surefire-plugin/src/main/java/org/apache/maven/plugin/surefire/SurefirePlugin.java @@ -31,6 +31,11 @@ import org.apache.maven.plugins.annotations.ResolutionScope; import org.apache.maven.surefire.suite.RunResult; +import java.nio.file.Files; +import java.nio.charset.Charset; +import java.io.IOException; + + import static org.apache.maven.plugin.surefire.SurefireHelper.reportExecution; /** @@ -584,6 +589,32 @@ public void setReportsDirectory( File reportsDirectory ) @Override public String getTest() { + // Path f = Paths.get( test ); + // if (Files.exists( f ) ) { + //try { + //List l = Files.readAllLines( f ); + //return String.join( "," , l ); + // } catch (IOException e ) { } + // } + + File f = new File( test ); + if ( f.exists() && !f.isDirectory ( ) ) + { + try + { + List l = Files.readAllLines( f.toPath(), Charset.defaultCharset( ) ); + StringBuilder sb = new StringBuilder( ); + for ( String s : l ) + { + sb.append( s + "," ); + } + String s = sb.toString( ); + return s.substring( 0 , s.length( ) - 1 ); + } + catch ( IOException e ) + { + } + } return test; } diff --git a/pom.xml b/pom.xml index 168dd0ee75..7295691f6f 100644 --- a/pom.xml +++ b/pom.xml @@ -516,7 +516,7 @@ org.codehaus.mojo animal-sniffer-maven-plugin - 1.17 + 1.18 maven-surefire-plugin diff --git a/surefire-api/src/main/java/org/apache/maven/surefire/util/DefaultRunOrderCalculator.java b/surefire-api/src/main/java/org/apache/maven/surefire/util/DefaultRunOrderCalculator.java index af15a93fc3..a3fc4e3ba6 100644 --- a/surefire-api/src/main/java/org/apache/maven/surefire/util/DefaultRunOrderCalculator.java +++ b/surefire-api/src/main/java/org/apache/maven/surefire/util/DefaultRunOrderCalculator.java @@ -131,10 +131,10 @@ public int compare( String o1, String o2 ) } else if ( MethodRunOrder.FLAKY_FINDING.equals( order ) ) { - String orderParam = System.getProperty( "flakyTestOrder" ); + String orderParam = System.getProperty( "test" ); if ( orderParam == null ) { - throw new IllegalStateException( "Please set system property flakyTestOrder to use flaky finding" ); + throw new IllegalStateException( "Please set system property test to use flaky finding" ); } final HashMap orders = new HashMap<>(); int i = 0; @@ -142,10 +142,10 @@ else if ( MethodRunOrder.FLAKY_FINDING.equals( order ) ) { orders.put( s, i ); i++; - if ( i > 2 ) - { - throw new UnsupportedOperationException( "This only supports 2 tests at a time for now." ); - } + // if ( i > 2 ) + // { + // throw new UnsupportedOperationException( "This only supports 2 tests at a time for now." ); + // } } return new Comparator() { @@ -227,7 +227,7 @@ private List> sortClassesBySpecifiedOrder( List> testClasses, LinkedList> ret = new LinkedList<>(); for ( String s : flakyTestOrder.split( "," ) ) { - String testClass = s.substring( s.indexOf( '(' ) + 1, s.length() - 1 ); + String testClass = s.substring( 0, s.indexOf( '#' ) ); Class c = classes.remove( testClass ); if ( c != null ) { From ab5a0813049c451fc565f52d45d44b93b9194d5a Mon Sep 17 00:00:00 2001 From: OrDTesters Date: Tue, 21 Jul 2020 06:19:12 +0000 Subject: [PATCH 06/24] Finalized features to run more than 2 tests, cleaned up arguments to run fixed order, tests to run can be passed in as a list from a file. --- README-bak.md | 74 ++++++++++++++++ README.md | 87 +++++++------------ .../maven/plugin/surefire/SurefirePlugin.java | 8 -- .../util/DefaultRunOrderCalculator.java | 56 ++++++++++-- .../maven/surefire/util/MethodRunOrder.java | 2 +- 5 files changed, 151 insertions(+), 76 deletions(-) create mode 100644 README-bak.md diff --git a/README-bak.md b/README-bak.md new file mode 100644 index 0000000000..83d4ec9829 --- /dev/null +++ b/README-bak.md @@ -0,0 +1,74 @@ + + +Maven + +Contributing to [Apache Maven Surefire](https://maven.apache.org/surefire/) +====================== + +[![Maven Central](https://img.shields.io/maven-central/v/org.apache.maven.surefire/surefire.svg?label=Maven%20Central&style=for-the-badge)](https://search.maven.org/#search%7Cgav%7C1%7Cg%3A%22org.apache.maven.plugins%22%20AND%20a%3A%22maven-surefire-plugin%22) +[![Apache License, Version 2.0, January 2004](https://img.shields.io/github/license/apache/maven.svg?label=License&style=for-the-badge)][license] + +[![CI](https://img.shields.io/badge/CI-Jenkins-blue.svg?style=for-the-badge)](https://jenkins-ci.org/) +[![Jenkins Status](https://img.shields.io/jenkins/s/https/builds.apache.org/job/maven-box/job/maven-surefire/job/master.svg?style=for-the-badge)][build] +[![Jenkins tests](https://img.shields.io/jenkins/t/https/builds.apache.org/job/maven-box/job/maven-surefire/job/master.svg?style=for-the-badge)][test-results] +[![Jenkins JaCoCo](https://img.shields.io/jenkins/coverage/jacoco/https/builds.apache.org/job/maven-box/job/maven-surefire/job/master.svg?style=for-the-badge&color=green)](https://builds.apache.org/job/maven-box/job/maven-surefire/job/master/lastBuild/jacoco/) + +[![Actions Status](https://github.com/apache/maven-surefire/workflows/GitHub%20CI/badge.svg?branch=master)](https://github.com/apache/maven-surefire/actions) + +# The Maven Community + +[![slack](https://img.shields.io/badge/slack-18/1138-pink.svg?style=for-the-badge)](https://the-asf.slack.com) +[![forks](https://img.shields.io/github/forks/apache/maven-surefire.svg?style=for-the-badge&label=Fork)](https://github.com/apache/maven-surefire/) + + +# Project Documentation + +[![Maven 3.0 Plugin API](https://img.shields.io/badge/maven%20site-documentation-blue.svg?style=for-the-badge)](https://maven.apache.org/surefire/) + +Usage of [maven-surefire-plugin], [maven-failsafe-plugin], [maven-surefire-report-plugin] + + +# Development Information + +Build the Surefire project using **Maven 3.1.0+** and **JDK 1.8**. + +* In order to run tests for a release check during the Vote, the following memory requirements are needed: + **(on Linux/Unix)** *export MAVEN_OPTS="-server -Xmx512m -XX:MetaspaceSize=128m -XX:MaxMetaspaceSize=384m -XX:+UseG1GC -XX:+UseStringDeduplication -XX:+TieredCompilation -XX:TieredStopAtLevel=1 -XX:SoftRefLRUPolicyMSPerMB=50 -Djava.awt.headless=true -Dhttps.protocols=TLSv1"* + **(on Windows)** *set MAVEN_OPTS="-server -Xmx256m -XX:MetaspaceSize=128m -XX:MaxMetaspaceSize=384m -XX:+UseG1GC -XX:+UseStringDeduplication -XX:+TieredCompilation -XX:TieredStopAtLevel=1 -XX:SoftRefLRUPolicyMSPerMB=50 -Djava.awt.headless=true -Dhttps.protocols=TLSv1"* +* In order to run the build with **JDK 9** **on Windows** (**on Linux/Unix modify system property jdk.home**): + *mvn install site site:stage -P reporting,run-its "-Djdk.home=e:\Program Files\Java\jdk9\"* +* In order to run the build with **JDK 11**: + *mvn install site site:stage -P reporting,run-its "-Djdk.home=e:\Program Files\Java\jdk11\"* + + +### Deploying web site + +See http://maven.apache.org/developers/website/deploy-component-reference-documentation.html + +[![Built with Maven](http://maven.apache.org/images/logos/maven-feather.png)](https://maven.apache.org/surefire/) + + +[license]: https://www.apache.org/licenses/LICENSE-2.0 +[build]: https://builds.apache.org/job/maven-box/job/maven-surefire/job/master/ +[test-results]: https://builds.apache.org/job/maven-box/job/maven-surefire/job/master/lastCompletedBuild/testReport/ +[Join us @ irc://freenode/maven]: https://www.irccloud.com/invite?channel=maven&hostname=irc.freenode.net&port=6697&ssl=1 +[Webchat with us @channel maven]: http://webchat.freenode.net/?channels=%23maven +[JIRA Change Log]: https://issues.apache.org/jira/browse/SUREFIRE/?selectedTab=com.atlassian.jira.jira-projects-plugin:changelog-panel +[maven-surefire-plugin]: https://maven.apache.org/surefire/maven-surefire-plugin/usage.html +[maven-failsafe-plugin]: https://maven.apache.org/surefire/maven-failsafe-plugin/usage.html +[maven-surefire-report-plugin]: https://maven.apache.org/surefire/maven-surefire-report-plugin/usage.html diff --git a/README.md b/README.md index 83d4ec9829..805a0bdb3b 100644 --- a/README.md +++ b/README.md @@ -1,74 +1,45 @@ - +## Example -Maven +``` +mvn test -Dtest=org.apache.dubbo.rpc.protocol.dubbo.DubboLazyConnectTest#testSticky1,org.apache.dubbo.rpc.protocol.dubbo.DubboProtocolTest#testDubboProtocol,org.apache.dubbo.rpc.protocol.dubbo.DubboProtocolTest#testDubboProtocolWithMina -Dsurefire.methodRunOrder=fixed +``` -Contributing to [Apache Maven Surefire](https://maven.apache.org/surefire/) -====================== +By specifying ```-Dsurefire.methodRunOrder=fixed``` Maven test will run the specifed tests in the order they appear in ```-Dtest```. +Running the command above will result in the tests running in the following order: -[![Maven Central](https://img.shields.io/maven-central/v/org.apache.maven.surefire/surefire.svg?label=Maven%20Central&style=for-the-badge)](https://search.maven.org/#search%7Cgav%7C1%7Cg%3A%22org.apache.maven.plugins%22%20AND%20a%3A%22maven-surefire-plugin%22) -[![Apache License, Version 2.0, January 2004](https://img.shields.io/github/license/apache/maven.svg?label=License&style=for-the-badge)][license] +1. org.apache.dubbo.rpc.protocol.dubbo.DubboLazyConnectTest.testSticky1 +2. org.apache.dubbo.rpc.protocol.dubbo.DubboProtocolTest.testDubboProtocol +3. org.apache.dubbo.rpc.protocol.dubbo.DubboProtocolTest.testDubboProtocolWithMina -[![CI](https://img.shields.io/badge/CI-Jenkins-blue.svg?style=for-the-badge)](https://jenkins-ci.org/) -[![Jenkins Status](https://img.shields.io/jenkins/s/https/builds.apache.org/job/maven-box/job/maven-surefire/job/master.svg?style=for-the-badge)][build] -[![Jenkins tests](https://img.shields.io/jenkins/t/https/builds.apache.org/job/maven-box/job/maven-surefire/job/master.svg?style=for-the-badge)][test-results] -[![Jenkins JaCoCo](https://img.shields.io/jenkins/coverage/jacoco/https/builds.apache.org/job/maven-box/job/maven-surefire/job/master.svg?style=for-the-badge&color=green)](https://builds.apache.org/job/maven-box/job/maven-surefire/job/master/lastBuild/jacoco/) +## Example with file -[![Actions Status](https://github.com/apache/maven-surefire/workflows/GitHub%20CI/badge.svg?branch=master)](https://github.com/apache/maven-surefire/actions) +``` +mvn test -Dtest=./path_to_file -Dsurefire.methodRunOrder=fixed +``` -# The Maven Community +By specifying ```-Dsurefire.methodRunOrder=fixed``` Maven test will run the specifed tests in the order that they appear in the file ```./path_to_file```. -[![slack](https://img.shields.io/badge/slack-18/1138-pink.svg?style=for-the-badge)](https://the-asf.slack.com) -[![forks](https://img.shields.io/github/forks/apache/maven-surefire.svg?style=for-the-badge&label=Fork)](https://github.com/apache/maven-surefire/) +Assume the content of ```./path_to_file``` are the following: +``` +org.apache.dubbo.rpc.protocol.dubbo.DubboLazyConnectTest#testSticky1 +org.apache.dubbo.rpc.protocol.dubbo.DubboProtocolTest#testDubboProtocol +org.apache.dubbo.rpc.protocol.dubbo.DubboProtocolTest#testDubboProtocolWithMina +``` -# Project Documentation +Then running the Maven command above will result in the tests running in the following order: -[![Maven 3.0 Plugin API](https://img.shields.io/badge/maven%20site-documentation-blue.svg?style=for-the-badge)](https://maven.apache.org/surefire/) +1. org.apache.dubbo.rpc.protocol.dubbo.DubboLazyConnectTest.testSticky1 +2. org.apache.dubbo.rpc.protocol.dubbo.DubboProtocolTest.testDubboProtocol +3. org.apache.dubbo.rpc.protocol.dubbo.DubboProtocolTest.testDubboProtocolWithMina -Usage of [maven-surefire-plugin], [maven-failsafe-plugin], [maven-surefire-report-plugin] +## Caveats -# Development Information +1. **Test methods from different test classes cannot interleave.** When test methods from various test classes interleave, all test methods from the first time the test class runs will be run then as well. E.g., If tests ClassA.testA, ClassB.testA, ClassA.testB are provided, then the run order will be ClassA.testA, ClassA.testB, ClassB.testA. -Build the Surefire project using **Maven 3.1.0+** and **JDK 1.8**. - -* In order to run tests for a release check during the Vote, the following memory requirements are needed: - **(on Linux/Unix)** *export MAVEN_OPTS="-server -Xmx512m -XX:MetaspaceSize=128m -XX:MaxMetaspaceSize=384m -XX:+UseG1GC -XX:+UseStringDeduplication -XX:+TieredCompilation -XX:TieredStopAtLevel=1 -XX:SoftRefLRUPolicyMSPerMB=50 -Djava.awt.headless=true -Dhttps.protocols=TLSv1"* - **(on Windows)** *set MAVEN_OPTS="-server -Xmx256m -XX:MetaspaceSize=128m -XX:MaxMetaspaceSize=384m -XX:+UseG1GC -XX:+UseStringDeduplication -XX:+TieredCompilation -XX:TieredStopAtLevel=1 -XX:SoftRefLRUPolicyMSPerMB=50 -Djava.awt.headless=true -Dhttps.protocols=TLSv1"* -* In order to run the build with **JDK 9** **on Windows** (**on Linux/Unix modify system property jdk.home**): - *mvn install site site:stage -P reporting,run-its "-Djdk.home=e:\Program Files\Java\jdk9\"* -* In order to run the build with **JDK 11**: - *mvn install site site:stage -P reporting,run-its "-Djdk.home=e:\Program Files\Java\jdk11\"* - - -### Deploying web site - -See http://maven.apache.org/developers/website/deploy-component-reference-documentation.html - -[![Built with Maven](http://maven.apache.org/images/logos/maven-feather.png)](https://maven.apache.org/surefire/) - - -[license]: https://www.apache.org/licenses/LICENSE-2.0 -[build]: https://builds.apache.org/job/maven-box/job/maven-surefire/job/master/ -[test-results]: https://builds.apache.org/job/maven-box/job/maven-surefire/job/master/lastCompletedBuild/testReport/ -[Join us @ irc://freenode/maven]: https://www.irccloud.com/invite?channel=maven&hostname=irc.freenode.net&port=6697&ssl=1 -[Webchat with us @channel maven]: http://webchat.freenode.net/?channels=%23maven -[JIRA Change Log]: https://issues.apache.org/jira/browse/SUREFIRE/?selectedTab=com.atlassian.jira.jira-projects-plugin:changelog-panel -[maven-surefire-plugin]: https://maven.apache.org/surefire/maven-surefire-plugin/usage.html -[maven-failsafe-plugin]: https://maven.apache.org/surefire/maven-failsafe-plugin/usage.html -[maven-surefire-report-plugin]: https://maven.apache.org/surefire/maven-surefire-report-plugin/usage.html +2. **FixMethodOrder annotations are ignored.** JUnit 4.11+ provides the annotation [FixMethodOrder](https://junit.org/junit4/javadoc/4.12/org/junit/FixMethodOrder.html) to control the ordering in which tests are run. Such annotations are ignored when this plugin is used. E.g., If tests ClassA.testB, ClassA.testA are provided and NAME_ASCENDING is used, then the run order will be ClassA.testB, ClassA.testA. diff --git a/maven-surefire-plugin/src/main/java/org/apache/maven/plugin/surefire/SurefirePlugin.java b/maven-surefire-plugin/src/main/java/org/apache/maven/plugin/surefire/SurefirePlugin.java index fd2e1a2a87..abd409efb9 100644 --- a/maven-surefire-plugin/src/main/java/org/apache/maven/plugin/surefire/SurefirePlugin.java +++ b/maven-surefire-plugin/src/main/java/org/apache/maven/plugin/surefire/SurefirePlugin.java @@ -589,14 +589,6 @@ public void setReportsDirectory( File reportsDirectory ) @Override public String getTest() { - // Path f = Paths.get( test ); - // if (Files.exists( f ) ) { - //try { - //List l = Files.readAllLines( f ); - //return String.join( "," , l ); - // } catch (IOException e ) { } - // } - File f = new File( test ); if ( f.exists() && !f.isDirectory ( ) ) { diff --git a/surefire-api/src/main/java/org/apache/maven/surefire/util/DefaultRunOrderCalculator.java b/surefire-api/src/main/java/org/apache/maven/surefire/util/DefaultRunOrderCalculator.java index a3fc4e3ba6..2269b4e3e9 100644 --- a/surefire-api/src/main/java/org/apache/maven/surefire/util/DefaultRunOrderCalculator.java +++ b/surefire-api/src/main/java/org/apache/maven/surefire/util/DefaultRunOrderCalculator.java @@ -32,6 +32,12 @@ import java.util.List; import java.util.Random; +import java.nio.file.Files; +import java.nio.charset.Charset; +import java.io.IOException; +import java.io.File; + + /** * Applies the final runorder of the tests * @@ -131,21 +137,17 @@ public int compare( String o1, String o2 ) } else if ( MethodRunOrder.FLAKY_FINDING.equals( order ) ) { - String orderParam = System.getProperty( "test" ); + String orderParam = parseFlakyTestOrder( System.getProperty( "test" ) ); if ( orderParam == null ) { - throw new IllegalStateException( "Please set system property test to use flaky finding" ); + throw new IllegalStateException( "Please set system property -Dtest to use fixed order" ); } final HashMap orders = new HashMap<>(); int i = 0; for ( String s : orderParam.split( "," ) ) { - orders.put( s, i ); + orders.put( changeTestNameFormatHashToParen( s ), i ); i++; - // if ( i > 2 ) - // { - // throw new UnsupportedOperationException( "This only supports 2 tests at a time for now." ); - // } } return new Comparator() { @@ -183,11 +185,21 @@ public int compare( String o1, String o2 ) } } + public String changeTestNameFormatHashToParen( String hashTestName ) + { + String[] nameSplit = hashTestName.split( "#" ); + String className = nameSplit[0]; + String testName = nameSplit[1]; + return testName + "(" + className + ")"; + } + private void orderTestClasses( List> testClasses, RunOrder runOrder ) { - if ( System.getProperty( "flakyTestOrder" ) != null ) + if ( System.getProperty( "surefire.methodRunOrder" ) != null + && System.getProperty( "surefire.methodRunOrder" ).toLowerCase().equals( "flakyfinding" ) ) { - List> sorted = sortClassesBySpecifiedOrder( testClasses, System.getProperty( "flakyTestOrder" ) ); + List> sorted + = sortClassesBySpecifiedOrder( testClasses, parseFlakyTestOrder( System.getProperty( "test" ) ) ); testClasses.clear(); testClasses.addAll( sorted ); } @@ -217,6 +229,32 @@ else if ( sortOrder != null ) } } + private String parseFlakyTestOrder( String s ) + { + if ( s != null && s != "" ) + { + File f = new File( s ); + if ( f.exists() && !f.isDirectory ( ) ) + { + try + { + List l = Files.readAllLines( f.toPath(), Charset.defaultCharset( ) ); + StringBuilder sb = new StringBuilder( ); + for ( String sd : l ) + { + sb.append( sd + "," ); + } + String sd = sb.toString( ); + return sd.substring( 0 , sd.length( ) - 1 ); + } + catch ( IOException e ) + { + } + } + } + return s; + } + private List> sortClassesBySpecifiedOrder( List> testClasses, String flakyTestOrder ) { HashMap> classes = new HashMap<>(); diff --git a/surefire-api/src/main/java/org/apache/maven/surefire/util/MethodRunOrder.java b/surefire-api/src/main/java/org/apache/maven/surefire/util/MethodRunOrder.java index 8d2d71e7a1..d41da320fb 100644 --- a/surefire-api/src/main/java/org/apache/maven/surefire/util/MethodRunOrder.java +++ b/surefire-api/src/main/java/org/apache/maven/surefire/util/MethodRunOrder.java @@ -28,7 +28,7 @@ public class MethodRunOrder public static final MethodRunOrder ALPHABETICAL = new MethodRunOrder( "alphabetical" ); public static final MethodRunOrder REVERSE_ALPHABETICAL = new MethodRunOrder( "reversealphabetical" ); public static final MethodRunOrder RANDOM = new MethodRunOrder( "random" ); - public static final MethodRunOrder FLAKY_FINDING = new MethodRunOrder( "flakyfinding" ); + public static final MethodRunOrder FLAKY_FINDING = new MethodRunOrder( "fixed" ); public static final MethodRunOrder DEFAULT = new MethodRunOrder( "default" ); private final String name; From 8f35368e9ec96506f9d0ea6fe48a0afcf676888f Mon Sep 17 00:00:00 2001 From: OrDTesters Date: Tue, 21 Jul 2020 06:45:56 +0000 Subject: [PATCH 07/24] Adding surefire-changing maven extension. --- README.md | 24 +- pom.xml | 1 + surefire-changing-maven-extension/.gitignore | 1 + surefire-changing-maven-extension/pom.xml | 87 ++++ .../fun/jvm/surefire/flaky/Configurator.java | 34 ++ .../flaky/KPLifecycleParticipant.java | 465 ++++++++++++++++++ 6 files changed, 606 insertions(+), 6 deletions(-) create mode 100644 surefire-changing-maven-extension/.gitignore create mode 100644 surefire-changing-maven-extension/pom.xml create mode 100644 surefire-changing-maven-extension/src/main/java/fun/jvm/surefire/flaky/Configurator.java create mode 100644 surefire-changing-maven-extension/src/main/java/fun/jvm/surefire/flaky/KPLifecycleParticipant.java diff --git a/README.md b/README.md index 805a0bdb3b..7d61a376d3 100644 --- a/README.md +++ b/README.md @@ -1,15 +1,27 @@ # Usage -To use the plugin, first run ```mvn install -DskipTests```. Then copy ```surefire-changing-maven-extension-1.0-SNAPSHOT.jar``` into your Maven installation's ```lib/ext``` directory. More information on how to use Maven extensions can be found [here](https://maven.apache.org/examples/maven-3-lifecycle-extensions.html#use-your-extension-in-your-build-s). +This repository is a fork of Maven Surefire that contains two main modifications. (1) A Maven extension to ensure that any Maven project one runs ```mvn test``` on will use this custom version of Surefire instead, and (2) the ability to control the ordering of tests run directly with Maven Surefire. + + +## Setup + +To use the plugin, perform the following steps. +1. Run ```mvn install -DskipTests``` in this directory +2. Copy ```surefire-changing-maven-extension/target/surefire-changing-maven-extension-1.0-SNAPSHOT.jar``` into your Maven installation's ```lib/ext``` directory + +The copying of the extension helps ensure that any project you run ```mvn test``` on will now use this custom version of Surefire and change certain settings (e.g., reuseForks to false) to prevent issues with fixing the ordering of tests. More information on how to use Maven extensions can be found [here](https://maven.apache.org/examples/maven-3-lifecycle-extensions.html#use-your-extension-in-your-build-s). Note that if you already have ```surefire-changing-maven-extension-1.0-SNAPSHOT.jar``` in your Maven installation's ```lib/ext``` you must first remove the jar before installing again. ## Example ``` -mvn test -Dtest=org.apache.dubbo.rpc.protocol.dubbo.DubboLazyConnectTest#testSticky1,org.apache.dubbo.rpc.protocol.dubbo.DubboProtocolTest#testDubboProtocol,org.apache.dubbo.rpc.protocol.dubbo.DubboProtocolTest#testDubboProtocolWithMina -Dsurefire.methodRunOrder=fixed +mvn test -Dsurefire.methodRunOrder=fixed \ +-Dtest=org.apache.dubbo.rpc.protocol.dubbo.DubboLazyConnectTest#testSticky1,\ +org.apache.dubbo.rpc.protocol.dubbo.DubboProtocolTest#testDubboProtocol,\ +org.apache.dubbo.rpc.protocol.dubbo.DubboProtocolTest#testDubboProtocolWithMina ``` -By specifying ```-Dsurefire.methodRunOrder=fixed``` Maven test will run the specifed tests in the order they appear in ```-Dtest```. -Running the command above will result in the tests running in the following order: +By specifying ```-Dsurefire.methodRunOrder=fixed``` Maven test will run the specifed tests in the order that they appear in ```-Dtest```. +Specifically, running the command above will result in the tests running in the following order: 1. org.apache.dubbo.rpc.protocol.dubbo.DubboLazyConnectTest.testSticky1 2. org.apache.dubbo.rpc.protocol.dubbo.DubboProtocolTest.testDubboProtocol @@ -40,6 +52,6 @@ Then running the Maven command above will result in the tests running in the fol ## Caveats -1. **Test methods from different test classes cannot interleave.** When test methods from various test classes interleave, all test methods from the first time the test class runs will be run then as well. E.g., If tests ClassA.testA, ClassB.testA, ClassA.testB are provided, then the run order will be ClassA.testA, ClassA.testB, ClassB.testA. +1. **Test methods from different test classes cannot interleave.** When test methods from various test classes interleave, all test methods from the first time the test class runs will run then as well. E.g., If tests ClassA.testA, ClassB.testA, ClassA.testB are provided, then the run order will be ClassA.testA, ClassA.testB, ClassB.testA. -2. **FixMethodOrder annotations are ignored.** JUnit 4.11+ provides the annotation [FixMethodOrder](https://junit.org/junit4/javadoc/4.12/org/junit/FixMethodOrder.html) to control the ordering in which tests are run. Such annotations are ignored when this plugin is used. E.g., If tests ClassA.testB, ClassA.testA are provided and NAME_ASCENDING is used, then the run order will be ClassA.testB, ClassA.testA. +2. **FixMethodOrder annotations are ignored.** JUnit 4.11+ provides the annotation [FixMethodOrder](https://junit.org/junit4/javadoc/4.12/org/junit/FixMethodOrder.html) to control the ordering in which tests are run. Such annotations are ignored when this plugin is used. E.g., If tests ClassA.testB, ClassA.testA are provided and FixMethodOrder is set to NAME_ASCENDING, then the run order will still be ClassA.testB, ClassA.testA. diff --git a/pom.xml b/pom.xml index 7295691f6f..a23ecf9b50 100644 --- a/pom.xml +++ b/pom.xml @@ -62,6 +62,7 @@ maven-surefire-report-plugin surefire-its surefire-shared-utils + surefire-changing-maven-extension diff --git a/surefire-changing-maven-extension/.gitignore b/surefire-changing-maven-extension/.gitignore new file mode 100644 index 0000000000..eb5a316cbd --- /dev/null +++ b/surefire-changing-maven-extension/.gitignore @@ -0,0 +1 @@ +target diff --git a/surefire-changing-maven-extension/pom.xml b/surefire-changing-maven-extension/pom.xml new file mode 100644 index 0000000000..7281be0943 --- /dev/null +++ b/surefire-changing-maven-extension/pom.xml @@ -0,0 +1,87 @@ + + 4.0.0 + + fun.jvm.surefire.flaky + surefire-changing-maven-extension + 1.0-SNAPSHOT + + UTF-8 + + + + MIT License + http://www.opensource.org/licenses/mit-license.php + repo + + + http://jonbell.net/ + + + winglam + Wing Lam + winglam2@illinois.edu + + + jbell + Jonathan Bell + bellj@gmu.edu + + + + + + org.apache.maven + maven-core + 3.6.3 + + + org.apache.maven + maven-embedder + 3.6.3 + + + + org.codehaus.plexus + plexus-component-annotations + 1.5.5 + + + junit + junit + + + + + junit + junit + 4.11 + test + + + + + + maven-compiler-plugin + 3.1 + + 1.7 + 1.7 + + + + + org.codehaus.plexus + plexus-component-metadata + 1.5.5 + + + + generate-metadata + + + + + + + diff --git a/surefire-changing-maven-extension/src/main/java/fun/jvm/surefire/flaky/Configurator.java b/surefire-changing-maven-extension/src/main/java/fun/jvm/surefire/flaky/Configurator.java new file mode 100644 index 0000000000..10b0a6fb26 --- /dev/null +++ b/surefire-changing-maven-extension/src/main/java/fun/jvm/surefire/flaky/Configurator.java @@ -0,0 +1,34 @@ +package fun.jvm.surefire.flaky; + +import org.apache.maven.execution.MavenSession; +import org.apache.maven.model.Plugin; +import org.apache.maven.model.PluginExecution; +import org.apache.maven.plugin.MojoFailureException; +import org.apache.maven.project.MavenProject; +import org.codehaus.plexus.util.xml.Xpp3Dom; + +public abstract class Configurator { + + protected MavenSession session; + public Configurator(MavenSession session) throws MojoFailureException { + this.session = session; + } + public abstract void applyConfiguration(MavenProject project, Plugin plugin, PluginExecution pluginExecution) throws MojoFailureException; + + public String getListenerClass(boolean isTestNG) { + return null; + } + + protected Plugin getOrAddPlugin(MavenProject project, String groupId, String artifactID, String version){ + Plugin ret = project.getPlugin(groupId+":"+artifactID); + if(ret == null){ + ret = new Plugin(); + ret.setGroupId(groupId); + ret.setArtifactId(artifactID); + ret.setVersion(version); + project.getBuild().addPlugin(ret); + project.getBuild().getPluginsAsMap().put(groupId+":"+artifactID,ret); + } + return ret; + } +} diff --git a/surefire-changing-maven-extension/src/main/java/fun/jvm/surefire/flaky/KPLifecycleParticipant.java b/surefire-changing-maven-extension/src/main/java/fun/jvm/surefire/flaky/KPLifecycleParticipant.java new file mode 100644 index 0000000000..1dd070121a --- /dev/null +++ b/surefire-changing-maven-extension/src/main/java/fun/jvm/surefire/flaky/KPLifecycleParticipant.java @@ -0,0 +1,465 @@ +package fun.jvm.surefire.flaky; + +import org.apache.maven.AbstractMavenLifecycleParticipant; +import org.apache.maven.MavenExecutionException; +import org.apache.maven.artifact.Artifact; +import org.apache.maven.artifact.repository.ArtifactRepository; +import org.apache.maven.artifact.repository.MavenArtifactRepository; +import org.apache.maven.execution.MavenSession; +import org.apache.maven.model.Dependency; +import org.apache.maven.model.Plugin; +import org.apache.maven.model.PluginExecution; +import org.apache.maven.model.Repository; +import org.apache.maven.plugin.MojoFailureException; +import org.apache.maven.project.MavenProject; +import org.apache.maven.project.artifact.PluginArtifact; +import org.apache.maven.settings.Mirror; +import org.codehaus.plexus.component.annotations.Component; +import org.codehaus.plexus.util.xml.Xpp3Dom; +import org.eclipse.aether.repository.RemoteRepository; + +import java.util.Collections; +import java.util.HashSet; +import java.util.LinkedList; + +@Component(role = AbstractMavenLifecycleParticipant.class, hint = "kp-ext") +public class KPLifecycleParticipant extends AbstractMavenLifecycleParticipant { + public static final String KP_VERSION = "1.0-SNAPSHOT"; + + private static final String ARGLINE_FLAGS = System.getenv("KP_ARGLINE"); + private static final String ADDL_TEST_DEPS = System.getenv("KP_DEPENDENCIES"); + private static final boolean FAIL_FAST = System.getenv("KP_FAIL_ON_FAILED_TEST") != null; + + public static boolean IGNORE_COMPILE_ERRORS = System.getenv("KP_IGNORE_COMPILE_ERRORS") != null; + private static final boolean RECORD_TESTS = System.getenv("KP_RECORD_TESTS") != null; + + private static final boolean DO_JACOCO = true; //System.getenv("KP_JACOCO") != null; + private static final boolean DO_PIT = System.getenv("KP_PIT") != null; + + private static final boolean FORK_PER_TEST = false; //System.getenv("KP_FORK_PER_TEST") != null && !DO_PIT; + static HashSet disabledPlugins = new HashSet(); + + static { + disabledPlugins.add("maven-enforcer-plugin"); + disabledPlugins.add("license-maven-plugin"); + disabledPlugins.add("maven-duplicate-finder-plugin"); + disabledPlugins.add("apache-rat-plugin"); + disabledPlugins.add("cobertura-maven-plugin"); + disabledPlugins.add("jacoco-maven-plugin"); + disabledPlugins.add("maven-dependency-versions-check-plugin"); + disabledPlugins.add("duplicate-finder-maven-plugin"); + } + + LinkedList configurators = new LinkedList<>(); + private MavenSession session; + + private void removeAnnoyingPlugins(MavenProject proj) { + LinkedList newPlugs = new LinkedList<>(); + for (Plugin p : proj.getBuildPlugins()) { + if (disabledPlugins.contains(p.getArtifactId())) { + System.out.println("Warning: KebabPizza disabling incompatible " + p.getGroupId() + ":" + p.getArtifactId() + " from " + proj.getArtifactId()); + } + else{ + newPlugs.add(p); + } + if (System.getProperty("diffcov.mysql") != null) { + //fix for checkstyle in evaluation + if (p.getArtifactId().equals("maven-antrun-plugin") && proj.getName().contains("checkstyle")) { + PluginExecution del = null; + for (PluginExecution pe : p.getExecutions()) { + if (pe.getId().equals("ant-phase-verify")) + del = pe; + } + if (del != null) + p.getExecutions().remove(del); + } + } + } + proj.getBuild().setPlugins(newPlugs); + + //Also, fix terrible junit deps + for (Dependency d : proj.getDependencies()) { + if ("junit".equals(d.getGroupId()) && "junit".equals(d.getArtifactId())) { + if ("4.2".equals(d.getVersion()) || "4.5".equals(d.getVersion()) || "4.4".equals(d.getVersion()) || "4.3".equals(d.getVersion()) || d.getVersion().startsWith("3")) + d.setVersion("4.6"); + } + } + + //also fix broken class files + if(IGNORE_COMPILE_ERRORS) { + for (Plugin p : proj.getBuildPlugins()) { + if (p.getArtifactId().equals("maven-compiler-plugin")) { + p.setVersion("3.8.0"); + for(PluginExecution pe : p.getExecutions()) { + Xpp3Dom config = (Xpp3Dom) pe.getConfiguration(); + if(config == null){ + config = new Xpp3Dom("configuration"); + pe.setConfiguration(config); + } + addOrSetValue("failOnError", "false", config); + addOrSetValue("compilerId", "eclipse", config); + addOrSetValue("fork", "true", config); + } + Dependency dep = new Dependency(); + dep.setArtifactId("plexus-compiler-eclipse"); + dep.setGroupId("org.codehaus.plexus"); + dep.setVersion("2.8.6"); + p.getDependencies().add(dep); + } + } + } + if(proj.getArtifactId().equals("spring-boot-loader-tools")){ + for(Plugin p : proj.getBuildPlugins()){ + if(p.getArtifactId().equals("maven-dependency-plugin")){ + for(PluginExecution pe : p.getExecutions()){ + if(pe.getId().equals("include-layout-jar")){ + pe.setPhase("prepare-package"); + } + } + } + } + } + if(proj.getArtifactId().equals("jboss-as-build")){ + //remove the ant run plugin + for(Plugin p : proj.getBuildPlugins()){ + if(p.getArtifactId().equals("maven-antrun-plugin")){ + p.getExecutions().clear(); + } + } + } + + //Also fix dead pluginrepos + + LinkedList reposToRemove = new LinkedList<>(); + LinkedList newRepos = new LinkedList<>(); + for (RemoteRepository r : proj.getRemotePluginRepositories()) { + if (r.getUrl().startsWith("http://build.eclipse.org")) + reposToRemove.add(r); + else if(r.getUrl().startsWith("http://repo.gradle") || r.getUrl().startsWith("http://repo.spring") || r.getUrl().startsWith("http://maven.spring")){ + reposToRemove.add(r); + RemoteRepository z = new RemoteRepository.Builder(r.getId(),r.getContentType(),r.getUrl().replace("http://","https://")).build(); + newRepos.add(z); + } + } + proj.getRemotePluginRepositories().addAll(newRepos); + if (reposToRemove.size() > 0) + proj.getRemotePluginRepositories().removeAll(reposToRemove); + + reposToRemove = new LinkedList<>(); + newRepos = new LinkedList<>(); + for (RemoteRepository r : proj.getRemoteProjectRepositories()) { + if (r.getUrl().startsWith("http://build.eclipse.org")) + reposToRemove.add(r); + else if(r.getUrl().startsWith("http://repo.gradle") || r.getUrl().startsWith("http://repo.spring") || r.getUrl().startsWith("http://maven.spring")){ + reposToRemove.add(r); + RemoteRepository z = new RemoteRepository.Builder(r.getId(),r.getContentType(),r.getUrl().replace("http://","https://")).build(); + newRepos.add(z); + } + } + proj.getRemoteProjectRepositories().addAll(newRepos); + if (reposToRemove.size() > 0) + proj.getRemoteProjectRepositories().removeAll(reposToRemove); + + + LinkedList areposToRemove = new LinkedList<>(); + LinkedList anewRepos = new LinkedList<>(); + + for (ArtifactRepository r : proj.getRemoteArtifactRepositories()) { + if (r.getUrl().startsWith("http://build.eclipse.org")) + areposToRemove.add(r); + else if (r.getUrl().startsWith("http://repo.gradle") || r.getUrl().startsWith("http://repo.spring") || r.getUrl().startsWith("http://maven.spring")) { + areposToRemove.add(r); + MavenArtifactRepository mar = (MavenArtifactRepository) r; + ArtifactRepository z = new MavenArtifactRepository(mar.getId(), mar.getUrl().replace("http://", "https://"), + mar.getLayout(), mar.getSnapshots(), mar.getReleases()); + anewRepos.add(z); + } + } + proj.getRemoteArtifactRepositories().addAll(anewRepos); + if (areposToRemove.size() > 0) + proj.getRemoteArtifactRepositories().removeAll(areposToRemove); + + } + + public static void addOrSetValue(String key, String value, Xpp3Dom parent){ + if(parent.getChild(key) != null) + parent.getChild(key).setValue(value); + else + { + Xpp3Dom v = new Xpp3Dom(key); + v.setValue(value); + parent.addChild(v); + } + } + public static Xpp3Dom addOrGetKey(String key, Xpp3Dom parent){ + if(parent.getChild(key) != null) + return parent.getChild(key); + else + { + Xpp3Dom v = new Xpp3Dom(key); + parent.addChild(v); + return v; + } + } + public void rewriteSurefireConfiguration(MavenProject project, Plugin p) throws MojoFailureException { + boolean testNG = false; + for (Dependency d : project.getDependencies()) { + if (d.getGroupId().equals("org.testng")) + testNG = true; + } + p.setVersion("3.0.0-FLAKYSNAPSHOT"); +// Dependency d = new Dependency(); +// d.setArtifactId("kp-test-listener"); +// d.setGroupId("edu.gmu.swe.smells"); +// d.setVersion("1.0-SNAPSHOT"); +// d.setScope("test"); +// project.getDependencies().add(d); +// if (ADDL_TEST_DEPS != null) { +// for (String dep : ADDL_TEST_DEPS.split(",")) { +// String[] dat = dep.split(":"); +// d = new Dependency(); +// d.setGroupId(dat[0]); +// d.setArtifactId(dat[1]); +// d.setVersion(dat[2]); +// d.setScope("test"); +// project.getDependencies().add(d); +// } +// } + + p.getDependencies().clear(); + for (PluginExecution pe : p.getExecutions()) { + + Xpp3Dom config = (Xpp3Dom) pe.getConfiguration(); + if (config == null) + config = new Xpp3Dom("configuration"); + rewriteSurefireExecutionConfig(config, testNG); + p.setConfiguration(config); + pe.setConfiguration(config); + for (Configurator c : configurators) + c.applyConfiguration(project, p, pe); + } + + } + + void rewriteSurefireExecutionConfig(Xpp3Dom config, boolean testNG) throws MojoFailureException { + + Xpp3Dom disableXmlReport = config.getChild("disableXmlReport"); + if(disableXmlReport != null) + { + disableXmlReport.setValue("false"); + } + Xpp3Dom argLine = config.getChild("argLine"); + if (argLine == null) { + argLine = new Xpp3Dom("argLine"); + argLine.setValue(""); + config.addChild(argLine); + } + argLine.setValue(argLine.getValue().replace("${surefireArgLine}", "")); + argLine.setValue(argLine.getValue().replace("'${jacocoArgLine}'", "")); + argLine.setValue(argLine.getValue().replace("${jacocoArgLine}", "")); + if (argLine != null && argLine.getValue().equals("${argLine}")) + argLine.setValue("'-XX:OnOutOfMemoryError=kill -9 %p' "); + else if (argLine != null) { + argLine.setValue("'-XX:OnOutOfMemoryError=kill -9 %p' " + argLine.getValue().replace("@{argLine}", "").replace("${argLine}", "").replace("${test.opts.coverage}", "")); + } + + //Now fix if we wanted jacoco or cobertura + if (ARGLINE_FLAGS != null) + argLine.setValue(ARGLINE_FLAGS + " " + argLine.getValue()); + Xpp3Dom parallel = config.getChild("parallel"); + if (parallel != null) + parallel.setValue("none"); + + fixForkMode(config, FORK_PER_TEST); + + + Xpp3Dom properties = config.getChild("properties"); + if (properties == null) { + properties = new Xpp3Dom("properties"); + config.addChild(properties); + } + if (RECORD_TESTS) { + Xpp3Dom prop = new Xpp3Dom("property"); + properties.addChild(prop); + Xpp3Dom propName = new Xpp3Dom("name"); + propName.setValue("listener"); + Xpp3Dom propValue = new Xpp3Dom("value"); + if (testNG) + propValue.setValue("edu.gmu.swe.kp.listener.TestNGExecutionListener"); + else + propValue.setValue("edu.gmu.swe.kp.listener.TestExecutionListener"); + + + prop.addChild(propName); + prop.addChild(propValue); + } + for (Configurator c : configurators) { + String listener = c.getListenerClass(testNG); + if (listener != null) { + Xpp3Dom prop = new Xpp3Dom("property"); + properties.addChild(prop); + Xpp3Dom propName = new Xpp3Dom("name"); + propName.setValue("listener"); + Xpp3Dom propValue = new Xpp3Dom("value"); + propValue.setValue(listener); + prop.addChild(propName); + prop.addChild(propValue); + } + } + + + Xpp3Dom testFailureIgnore = config.getChild("testFailureIgnore"); + if (testFailureIgnore != null) { + testFailureIgnore.setValue((FAIL_FAST ? "false" : "true")); + } else { + testFailureIgnore = new Xpp3Dom("testFailureIgnore"); + testFailureIgnore.setValue((FAIL_FAST ? "false" : "true")); + config.addChild(testFailureIgnore); + } + + Xpp3Dom vars = config.getChild("systemPropertyVariables"); + if (vars == null) { + vars = new Xpp3Dom("systemPropertyVariables"); + config.addChild(vars); + } + } + + void fixForkMode(Xpp3Dom config, boolean forkPerTest) { + Xpp3Dom forkMode = config.getChild("forkMode"); + boolean isSetToFork = false; + if (forkPerTest) { + if (forkMode != null) + forkMode.setValue("perTest"); + else { + Xpp3Dom forkCount = config.getChild("forkCount"); + if (forkCount != null) + forkCount.setValue("1"); + else { + forkCount = new Xpp3Dom("forkCount"); + forkCount.setValue("1"); + config.addChild(forkCount); + } + Xpp3Dom reuseForks = config.getChild("reuseForks"); + if (reuseForks != null) + reuseForks.setValue("false"); + else { + reuseForks = new Xpp3Dom("reuseForks"); + reuseForks.setValue("false"); + config.addChild(reuseForks); + } + } + return; + } else { + //Want to NOT fork per-test + if (forkMode != null) { + forkMode.setValue("once"); + isSetToFork = true; + } + + Xpp3Dom forkCount = config.getChild("forkCount"); + if (forkCount != null && !forkCount.getValue().equals("1")) { + isSetToFork = true; + forkCount.setValue("1"); + } + + Xpp3Dom reuseForks = config.getChild("reuseForks"); + if (reuseForks != null && reuseForks.getValue().equals("false")) { + reuseForks.setValue("true"); + } + + if (!isSetToFork) { + forkCount = new Xpp3Dom("forkCount"); + forkCount.setValue("1"); + config.addChild(forkCount); + } + } + } + + @Override + public void afterSessionStart(MavenSession session) throws MavenExecutionException { + super.afterSessionStart(session); + Mirror springSnaps = new Mirror(); + springSnaps.setMirrorOf("spring-snapshots"); + springSnaps.setUrl("https://repo.spring.io/snapshot"); + session.getRequest().getMirrors().add(springSnaps); + } + + @Override + public void afterProjectsRead(MavenSession session) throws MavenExecutionException { + + this.session = session; +// try { +// if (DO_JACOCO) +// configurators.add(new JacocoConfigurator(session)); +// if (DO_PIT) +// configurators.add(new PitConfigurator(session)); +// } catch (MojoFailureException ex) { +// throw new MavenExecutionException("Unable to create KP configurator", ex); +// } + + //Find out which is the last run of tests + Plugin lastRunOfSurefireOrFailsafe = null; + for (MavenProject p : session.getProjects()) { + for (Plugin o : p.getBuildPlugins()) { + if ((o.getArtifactId().equals("maven-surefire-plugin") || o.getArtifactId().equals("maven-failsafe-plugin")) + && o.getGroupId().equals("org.apache.maven.plugins")) { + lastRunOfSurefireOrFailsafe = o; + } + } + + } + for (MavenProject p : session.getProjects()) { + removeAnnoyingPlugins(p); +// addDependencyPlugin(p); + try { + LinkedList toModify = new LinkedList<>(p.getBuildPlugins()); + for(Artifact pa : p.getPluginArtifacts()){ + if(pa.getArtifactId().equals("maven-surefire-plugin") || pa.getArtifactId().equals("maven-failsafe-plugin")){ + pa.setVersion("3.0.0-FLAKYSNAPSHOT"); + } + } + for (Plugin o : toModify) { + if ((o.getArtifactId().equals("maven-surefire-plugin") && o.getGroupId().equals("org.apache.maven.plugins")) || (o.getArtifactId().equals("maven-failsafe-plugin") && o.getGroupId().equals("org.apache.maven.plugins"))) { + rewriteSurefireConfiguration(p, o); + } + } + } catch (MojoFailureException e) { + e.printStackTrace(); + } + } + + } + + private void fixServers(MavenProject p){ + for(Repository r : p.getRepositories()){ +// if(r.getUrl().startsWith("http://spring")) + } + } + private void addDependencyPlugin(MavenProject p) { + Plugin depPlugin = null; + for(Plugin o : p.getBuildPlugins()){ + if(o.getArtifactId().equals("maven-dependency-plugin") && o.getGroupId().equals("org.apache.maven.plugins")){ + depPlugin = o; + } + } + if(depPlugin == null){ + depPlugin = new Plugin(); + depPlugin.setArtifactId("maven-dependency-plugin"); + depPlugin.setGroupId("org.apache.maven.plugins"); + depPlugin.setVersion("3.1.2"); + p.getBuild().addPlugin(depPlugin); + } + PluginExecution ex = new PluginExecution(); + Xpp3Dom config = new Xpp3Dom("configuration"); + Xpp3Dom outputProperty = new Xpp3Dom("outputProperty"); + outputProperty.setValue("flakyTestClasspath"); + config.addChild(outputProperty); + ex.setConfiguration(config); + ex.setId("kp-cp-build"); + ex.setPhase("generate-sources"); + ex.setGoals(Collections.singletonList("build-classpath")); + depPlugin.addExecution(ex); + + } + +} From 6fc2515978f9827a50265ddb40f924cfe46c7c96 Mon Sep 17 00:00:00 2001 From: OrDTesters Date: Tue, 21 Jul 2020 06:52:27 +0000 Subject: [PATCH 08/24] Adding todos. --- README.md | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/README.md b/README.md index 7d61a376d3..90db79a928 100644 --- a/README.md +++ b/README.md @@ -55,3 +55,13 @@ Then running the Maven command above will result in the tests running in the fol 1. **Test methods from different test classes cannot interleave.** When test methods from various test classes interleave, all test methods from the first time the test class runs will run then as well. E.g., If tests ClassA.testA, ClassB.testA, ClassA.testB are provided, then the run order will be ClassA.testA, ClassA.testB, ClassB.testA. 2. **FixMethodOrder annotations are ignored.** JUnit 4.11+ provides the annotation [FixMethodOrder](https://junit.org/junit4/javadoc/4.12/org/junit/FixMethodOrder.html) to control the ordering in which tests are run. Such annotations are ignored when this plugin is used. E.g., If tests ClassA.testB, ClassA.testA are provided and FixMethodOrder is set to NAME_ASCENDING, then the run order will still be ClassA.testB, ClassA.testA. + +## TODOs + +The following are features that we would like to have but are yet to be supported. + +1. Randomize every test in a class + - Randomize with seed, prints the seed somewhere +2. Have surefire reports save the order in which the test classes are run +3. Allow one to get just the test order list without running tests +4. Reverse mode From 4954fd11d4b6b9536b845e1e1d35d6d140025887 Mon Sep 17 00:00:00 2001 From: OrDTesters Date: Tue, 21 Jul 2020 06:54:27 +0000 Subject: [PATCH 09/24] Cleaning readme. --- README.md | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 90db79a928..97855053c6 100644 --- a/README.md +++ b/README.md @@ -1,11 +1,15 @@ # Usage -This repository is a fork of Maven Surefire that contains two main modifications. (1) A Maven extension to ensure that any Maven project one runs ```mvn test``` on will use this custom version of Surefire instead, and (2) the ability to control the ordering of tests run directly with Maven Surefire. +This repository is a fork of Maven Surefire that contains two main modifications. + +1. A Maven extension to ensure that any Maven project one runs ```mvn test``` on will use this custom version of Surefire instead, and +2. the ability to control the ordering of tests run directly with Maven Surefire. ## Setup To use the plugin, perform the following steps. + 1. Run ```mvn install -DskipTests``` in this directory 2. Copy ```surefire-changing-maven-extension/target/surefire-changing-maven-extension-1.0-SNAPSHOT.jar``` into your Maven installation's ```lib/ext``` directory From 65a6f64f06a3584d816441ae25c82446b3516b5e Mon Sep 17 00:00:00 2001 From: OrDTesters Date: Tue, 21 Jul 2020 06:57:55 +0000 Subject: [PATCH 10/24] Removing extra space. --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 97855053c6..a14c5d1196 100644 --- a/README.md +++ b/README.md @@ -18,7 +18,7 @@ The copying of the extension helps ensure that any project you run ```mvn test`` ## Example ``` -mvn test -Dsurefire.methodRunOrder=fixed \ +mvn test -Dsurefire.methodRunOrder=fixed \ -Dtest=org.apache.dubbo.rpc.protocol.dubbo.DubboLazyConnectTest#testSticky1,\ org.apache.dubbo.rpc.protocol.dubbo.DubboProtocolTest#testDubboProtocol,\ org.apache.dubbo.rpc.protocol.dubbo.DubboProtocolTest#testDubboProtocolWithMina From 12c5ae8390ea4ca794432b6048ec7bc0de24c0ed Mon Sep 17 00:00:00 2001 From: OrDTesters Date: Mon, 27 Jul 2020 23:03:51 +0000 Subject: [PATCH 11/24] Improving readme. --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index a14c5d1196..bbe51d2308 100644 --- a/README.md +++ b/README.md @@ -64,7 +64,7 @@ Then running the Maven command above will result in the tests running in the fol The following are features that we would like to have but are yet to be supported. -1. Randomize every test in a class +1. Randomize every test in a class (mostly done by Jon already) - Randomize with seed, prints the seed somewhere 2. Have surefire reports save the order in which the test classes are run 3. Allow one to get just the test order list without running tests From bd1946fe399651dd3e2f97f2a51d5294e5d24ad2 Mon Sep 17 00:00:00 2001 From: Wing Lam Date: Fri, 9 Oct 2020 03:08:22 +0000 Subject: [PATCH 12/24] adding issues. --- issues.md | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 issues.md diff --git a/issues.md b/issues.md new file mode 100644 index 0000000000..10b6ec9927 --- /dev/null +++ b/issues.md @@ -0,0 +1,6 @@ +Fails to run in expected order according to David when he was inspecting UD tests for OOPSLA. + +Script for reproducing: +http://mir.cs.illinois.edu/winglam/personal/surefire-method-sorting-bug.sh +Log for my script: +http://mir.cs.illinois.edu/winglam/personal/surefire-method-sorting-bug.log \ No newline at end of file From 2302c4109c1ad1db8b6cd1d3c91fa6d6028c625c Mon Sep 17 00:00:00 2001 From: Wing Lam Date: Fri, 9 Oct 2020 20:39:17 +0000 Subject: [PATCH 13/24] Fixing issue with fixing method order. --- README.md | 20 ++++++++++++++----- .../api/util/DefaultRunOrderCalculator.java | 2 +- 2 files changed, 16 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index f5e801b12c..4e276eefa5 100644 --- a/README.md +++ b/README.md @@ -53,6 +53,17 @@ Then running the Maven command above will result in the tests running in the fol 3. org.apache.dubbo.rpc.protocol.dubbo.DubboProtocolTest.testDubboProtocolWithMina +## Random with seed + +This change has been merged to [apache/maven-surefire](https://github.com/apache/maven-surefire/pull/309). + +Specifically, Surefire will: + +1. Output of the random seed used to generate a particular random test order when -Dsurefire.runOrder=random or -Dfailsafe.runOrder=random is set +2. Replay a previously observed random test order by setting -Dsurefire.runOrder.random.seed and -Dfailsafe.runOrder.random.seed to the seed that observed the random test order + +Some tests were added to Surefire for this feature. These tests ensure that the setting of the same random seeds do create the same test orders and different random seeds do create different test orders. Note that the inherent randomness of the orders does mean that the tests can be flaky (nondeterministically pass or fail without changes to the code). The current tests have a rate of 0.4% (1/3)^5 of failing. Increasing the number of tests (3) or the number of times to loop (5) would decrease the odds of the tests failing. + ## Caveats 1. **Test methods from different test classes cannot interleave.** When test methods from various test classes interleave, all test methods from the first time the test class runs will run then as well. E.g., If tests ClassA.testA, ClassB.testA, ClassA.testB are provided, then the run order will be ClassA.testA, ClassA.testB, ClassB.testA. @@ -63,8 +74,7 @@ Then running the Maven command above will result in the tests running in the fol The following are features that we would like to have but are yet to be supported. -1. Randomize every test in a class (mostly done by Jon already) - - Randomize with seed, prints the seed somewhere -2. Have surefire reports save the order in which the test classes are run -3. Allow one to get just the test order list without running tests -4. Reverse mode +1. Have surefire reports save the order in which the test classes are run +2. Allow one to get just the test order list without running tests +3. Reverse mode +4. Have Surefire fix order-dependent tests observed in last run diff --git a/surefire-api/src/main/java/org/apache/maven/surefire/api/util/DefaultRunOrderCalculator.java b/surefire-api/src/main/java/org/apache/maven/surefire/api/util/DefaultRunOrderCalculator.java index bc620bf535..0664764054 100644 --- a/surefire-api/src/main/java/org/apache/maven/surefire/api/util/DefaultRunOrderCalculator.java +++ b/surefire-api/src/main/java/org/apache/maven/surefire/api/util/DefaultRunOrderCalculator.java @@ -200,7 +200,7 @@ public String changeTestNameFormatHashToParen( String hashTestName ) private void orderTestClasses( List> testClasses, RunOrder runOrder ) { if ( System.getProperty( "surefire.methodRunOrder" ) != null - && System.getProperty( "surefire.methodRunOrder" ).toLowerCase().equals( "flakyfinding" ) ) + && System.getProperty( "surefire.methodRunOrder" ).toLowerCase().equals( "fixed" ) ) { List> sorted = sortClassesBySpecifiedOrder( testClasses, parseFlakyTestOrder( System.getProperty( "test" ) ) ); From 87b62ce90b279ede3797500500f6ddbce13f8f39 Mon Sep 17 00:00:00 2001 From: Wing Lam Date: Fri, 9 Oct 2020 20:50:50 +0000 Subject: [PATCH 14/24] Removing remaining randomSeeds. --- .../org/apache/maven/plugin/failsafe/IntegrationTestMojo.java | 4 ---- .../java/org/apache/maven/plugin/surefire/SurefirePlugin.java | 4 ---- .../org/apache/maven/surefire/booter/BooterConstants.java | 1 - .../org/apache/maven/surefire/booter/BooterDeserializer.java | 1 - 4 files changed, 10 deletions(-) diff --git a/maven-failsafe-plugin/src/main/java/org/apache/maven/plugin/failsafe/IntegrationTestMojo.java b/maven-failsafe-plugin/src/main/java/org/apache/maven/plugin/failsafe/IntegrationTestMojo.java index 83994d7829..ab3b364aff 100644 --- a/maven-failsafe-plugin/src/main/java/org/apache/maven/plugin/failsafe/IntegrationTestMojo.java +++ b/maven-failsafe-plugin/src/main/java/org/apache/maven/plugin/failsafe/IntegrationTestMojo.java @@ -314,10 +314,6 @@ public class IntegrationTestMojo @Parameter( property = "failsafe.methodRunOrder", defaultValue = "default" ) private String methodRunOrder; - @Parameter( property = "failsafe.seed" ) - private long randomSeed; - - /** * Sets the random seed that will be used to order the tests if {@code failsafe.runOrder} is set to {@code random}. *
diff --git a/maven-surefire-plugin/src/main/java/org/apache/maven/plugin/surefire/SurefirePlugin.java b/maven-surefire-plugin/src/main/java/org/apache/maven/plugin/surefire/SurefirePlugin.java index cf1d70752f..93a4b6a6cf 100644 --- a/maven-surefire-plugin/src/main/java/org/apache/maven/plugin/surefire/SurefirePlugin.java +++ b/maven-surefire-plugin/src/main/java/org/apache/maven/plugin/surefire/SurefirePlugin.java @@ -297,14 +297,10 @@ public class SurefirePlugin @Parameter( property = "surefire.runOrder", defaultValue = "filesystem" ) private String runOrder; - //TODO docs @Parameter( property = "surefire.methodRunOrder", defaultValue = "default" ) private String methodRunOrder; - @Parameter( property = "surefire.seed" ) - private long randomSeed; - /** * Sets the random seed that will be used to order the tests if {@code surefire.runOrder} is set to {@code random}. *
diff --git a/surefire-booter/src/main/java/org/apache/maven/surefire/booter/BooterConstants.java b/surefire-booter/src/main/java/org/apache/maven/surefire/booter/BooterConstants.java index 2649eb7e0e..3492fb6bea 100644 --- a/surefire-booter/src/main/java/org/apache/maven/surefire/booter/BooterConstants.java +++ b/surefire-booter/src/main/java/org/apache/maven/surefire/booter/BooterConstants.java @@ -49,7 +49,6 @@ private BooterConstants() public static final String RUN_ORDER_RANDOM_SEED = "runOrderRandomSeed"; public static final String RUN_STATISTICS_FILE = "runStatisticsFile"; public static final String METHOD_RUN_ORDER = "methodRunOrder"; - public static final String RANDOM_SEED = "randomSeed"; public static final String TEST_SUITE_XML_FILES = "testSuiteXmlFiles"; public static final String PROVIDER_CONFIGURATION = "providerConfiguration"; public static final String FORKTESTSET = "forkTestSet"; diff --git a/surefire-booter/src/main/java/org/apache/maven/surefire/booter/BooterDeserializer.java b/surefire-booter/src/main/java/org/apache/maven/surefire/booter/BooterDeserializer.java index 9d4acb33af..0caa54e5bf 100644 --- a/surefire-booter/src/main/java/org/apache/maven/surefire/booter/BooterDeserializer.java +++ b/surefire-booter/src/main/java/org/apache/maven/surefire/booter/BooterDeserializer.java @@ -105,7 +105,6 @@ public ProviderConfiguration deserialize() final Long runOrderRandomSeed = properties.getLongProperty( RUN_ORDER_RANDOM_SEED ); final String runStatisticsFile = properties.getProperty( RUN_STATISTICS_FILE ); final String methodRunOrder = properties.getProperty( METHOD_RUN_ORDER ); - final long randomSeed = properties.getLongProperty( RANDOM_SEED ); final int rerunFailingTestsCount = properties.getIntProperty( RERUN_FAILING_TESTS_COUNT ); From 2490a4b151cb34d2a6738ea6e808c345ccc0b17f Mon Sep 17 00:00:00 2001 From: Wing Lam Date: Fri, 9 Oct 2020 21:29:48 +0000 Subject: [PATCH 15/24] Improving readme. --- README.md | 22 ++++++++++++++-------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index 4e276eefa5..1ea355fd4c 100644 --- a/README.md +++ b/README.md @@ -9,7 +9,7 @@ This repository is a fork of Maven Surefire that contains two main modifications To use the plugin, perform the following steps. -1. Run ```mvn install -DskipTests``` in this directory +1. Run ```mvn install -DskipTests -Drat.skip``` in this directory 2. Copy ```surefire-changing-maven-extension/target/surefire-changing-maven-extension-1.0-SNAPSHOT.jar``` into your Maven installation's ```lib/ext``` directory The copying of the extension helps ensure that any project you run ```mvn test``` on will now use this custom version of Surefire and change certain settings (e.g., reuseForks to false) to prevent issues with fixing the ordering of tests. More information on how to use Maven extensions can be found [here](https://maven.apache.org/examples/maven-3-lifecycle-extensions.html#use-your-extension-in-your-build-s). Note that if you already have ```surefire-changing-maven-extension-1.0-SNAPSHOT.jar``` in your Maven installation's ```lib/ext``` you must first remove the jar before installing again. @@ -20,7 +20,8 @@ The copying of the extension helps ensure that any project you run ```mvn test`` mvn test -Dsurefire.methodRunOrder=fixed \ -Dtest=org.apache.dubbo.rpc.protocol.dubbo.DubboLazyConnectTest#testSticky1,\ org.apache.dubbo.rpc.protocol.dubbo.DubboProtocolTest#testDubboProtocol,\ -org.apache.dubbo.rpc.protocol.dubbo.DubboProtocolTest#testDubboProtocolWithMina +org.apache.dubbo.rpc.protocol.dubbo.DubboProtocolTest#testDubboProtocolWithMina\ +-pl dubbo-rpc/dubbo-rpc-dubbo ``` By specifying ```-Dsurefire.methodRunOrder=fixed``` Maven test will run the specifed tests in the order that they appear in ```-Dtest```. @@ -33,12 +34,12 @@ Specifically, running the command above will result in the tests running in the ## Example with file ``` -mvn test -Dtest=./path_to_file -Dsurefire.methodRunOrder=fixed +mvn test -Dtest=path_to_file -Dsurefire.methodRunOrder=fixed -pl dubbo-rpc/dubbo-rpc-dubbo ``` -By specifying ```-Dsurefire.methodRunOrder=fixed``` Maven test will run the specifed tests in the order that they appear in the file ```./path_to_file```. +By specifying ```-Dsurefire.methodRunOrder=fixed``` Maven test will run the specifed tests in the order that they appear in the file ```path_to_file```. -Assume the content of ```./path_to_file``` are the following: +Assume the content of ```path_to_file``` are the following: ``` org.apache.dubbo.rpc.protocol.dubbo.DubboLazyConnectTest#testSticky1 @@ -55,12 +56,17 @@ Then running the Maven command above will result in the tests running in the fol ## Random with seed -This change has been merged to [apache/maven-surefire](https://github.com/apache/maven-surefire/pull/309). +This specific feature has been merged to [apache/maven-surefire](https://github.com/apache/maven-surefire/pull/309). The other features in this repository will be submitted to Surefire soon. Specifically, Surefire will: -1. Output of the random seed used to generate a particular random test order when -Dsurefire.runOrder=random or -Dfailsafe.runOrder=random is set -2. Replay a previously observed random test order by setting -Dsurefire.runOrder.random.seed and -Dfailsafe.runOrder.random.seed to the seed that observed the random test order +1. Output of the random seed used to generate a particular random test order when `-Dsurefire.runOrder.random.seed` and `-Dfailsafe.runOrder.random.seed` are not set. To get the seed, look for the following in the output: +``` +Tests will run in random order. To reproduce ordering use flag -Dsurefire.runOrder.random.seed=28421961536740501 +``` +2. Replay a previously observed random test order by setting `-Dsurefire.runOrder.random.seed` and `-Dfailsafe.runOrder.random.seed` to the seed that observed the random test order + +Note that the seed will control the random of both test classes and test methods if both are set to random. E.g., ```-Dsurefire.runOrder=random -Dsurefire.methodRunOrder=random -Dsurefire.runOrder.random.seed=28421961536740501``` will randomize both test classes and test methods with ```28421961536740501``` as the seed. It is currently not possible to set the seed of the test class and test method to be different. Some tests were added to Surefire for this feature. These tests ensure that the setting of the same random seeds do create the same test orders and different random seeds do create different test orders. Note that the inherent randomness of the orders does mean that the tests can be flaky (nondeterministically pass or fail without changes to the code). The current tests have a rate of 0.4% (1/3)^5 of failing. Increasing the number of tests (3) or the number of times to loop (5) would decrease the odds of the tests failing. From 320efe9f660039be781b033a714de5327c05f911 Mon Sep 17 00:00:00 2001 From: Wing Lam Date: Fri, 9 Oct 2020 21:40:09 +0000 Subject: [PATCH 16/24] Updating issues. --- issues.md | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/issues.md b/issues.md index 10b6ec9927..8d987f0433 100644 --- a/issues.md +++ b/issues.md @@ -1,6 +1 @@ -Fails to run in expected order according to David when he was inspecting UD tests for OOPSLA. - -Script for reproducing: -http://mir.cs.illinois.edu/winglam/personal/surefire-method-sorting-bug.sh -Log for my script: -http://mir.cs.illinois.edu/winglam/personal/surefire-method-sorting-bug.log \ No newline at end of file +There are no known issues at this point. From e2ed417b3de1a5e5bc20b8fd1a2fe9a38324da64 Mon Sep 17 00:00:00 2001 From: Wing Lam Date: Wed, 28 Oct 2020 22:06:34 +0000 Subject: [PATCH 17/24] Updating readme to clarify usage of path_to_file --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 1ea355fd4c..21a7d38010 100644 --- a/README.md +++ b/README.md @@ -37,7 +37,7 @@ Specifically, running the command above will result in the tests running in the mvn test -Dtest=path_to_file -Dsurefire.methodRunOrder=fixed -pl dubbo-rpc/dubbo-rpc-dubbo ``` -By specifying ```-Dsurefire.methodRunOrder=fixed``` Maven test will run the specifed tests in the order that they appear in the file ```path_to_file```. +By specifying ```-Dsurefire.methodRunOrder=fixed``` Maven test will run the specifed tests in the order that they appear in the file ```path_to_file```. Note that the ```path_to_file``` should be an **absolute** path (e.g., ```/home/user/project/test-list```). Assume the content of ```path_to_file``` are the following: From 8162cb58c2553da1b6c2f1dc4d3b5e4c84137cf8 Mon Sep 17 00:00:00 2001 From: winglam Date: Tue, 1 Dec 2020 17:35:34 -0600 Subject: [PATCH 18/24] Updating README instructions about branch --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 21a7d38010..678f71a13d 100644 --- a/README.md +++ b/README.md @@ -7,7 +7,7 @@ This repository is a fork of Maven Surefire that contains two main modifications ## Setup -To use the plugin, perform the following steps. +To use the plugin, please ensure you are on the `test-method-sorting` branch and then perform the following steps. 1. Run ```mvn install -DskipTests -Drat.skip``` in this directory 2. Copy ```surefire-changing-maven-extension/target/surefire-changing-maven-extension-1.0-SNAPSHOT.jar``` into your Maven installation's ```lib/ext``` directory From e2184c622834a33a6fe2a11ef70fd8949918ace2 Mon Sep 17 00:00:00 2001 From: winglam Date: Sun, 6 Dec 2020 16:49:33 -0600 Subject: [PATCH 19/24] Adding information that may help locate Maven installation --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 678f71a13d..499a020880 100644 --- a/README.md +++ b/README.md @@ -10,7 +10,7 @@ This repository is a fork of Maven Surefire that contains two main modifications To use the plugin, please ensure you are on the `test-method-sorting` branch and then perform the following steps. 1. Run ```mvn install -DskipTests -Drat.skip``` in this directory -2. Copy ```surefire-changing-maven-extension/target/surefire-changing-maven-extension-1.0-SNAPSHOT.jar``` into your Maven installation's ```lib/ext``` directory +2. Copy ```surefire-changing-maven-extension/target/surefire-changing-maven-extension-1.0-SNAPSHOT.jar``` into your Maven installation's ```lib/ext``` directory. This [StackOverflow post](https://stackoverflow.com/a/39479104) may help indicate where your Maven installation is located The copying of the extension helps ensure that any project you run ```mvn test``` on will now use this custom version of Surefire and change certain settings (e.g., reuseForks to false) to prevent issues with fixing the ordering of tests. More information on how to use Maven extensions can be found [here](https://maven.apache.org/examples/maven-3-lifecycle-extensions.html#use-your-extension-in-your-build-s). Note that if you already have ```surefire-changing-maven-extension-1.0-SNAPSHOT.jar``` in your Maven installation's ```lib/ext``` you must first remove the jar before installing again. From f191e2378b098a9ac558bd3141249a725108e1fa Mon Sep 17 00:00:00 2001 From: Wing Lam Date: Sat, 20 Feb 2021 02:24:19 +0000 Subject: [PATCH 20/24] Fix for controlling test class ordering. --- .../api/util/DefaultRunOrderCalculator.java | 81 +++++++++++++------ 1 file changed, 56 insertions(+), 25 deletions(-) diff --git a/surefire-api/src/main/java/org/apache/maven/surefire/api/util/DefaultRunOrderCalculator.java b/surefire-api/src/main/java/org/apache/maven/surefire/api/util/DefaultRunOrderCalculator.java index 0664764054..fc204ef9d0 100644 --- a/surefire-api/src/main/java/org/apache/maven/surefire/api/util/DefaultRunOrderCalculator.java +++ b/surefire-api/src/main/java/org/apache/maven/surefire/api/util/DefaultRunOrderCalculator.java @@ -32,6 +32,7 @@ import java.util.Collections; import java.util.Comparator; import java.util.HashMap; +import java.util.LinkedHashMap; import java.util.LinkedHashSet; import java.util.LinkedList; import java.util.List; @@ -146,39 +147,60 @@ else if ( MethodRunOrder.FLAKY_FINDING.equals( order ) ) { throw new IllegalStateException( "Please set system property -Dtest to use fixed order" ); } - final HashMap orders = new HashMap<>(); - int i = 0; + final LinkedHashMap> orders = new LinkedHashMap<>(); for ( String s : orderParam.split( "," ) ) { - orders.put( changeTestNameFormatHashToParen( s ), i ); - i++; + String[] nameSplit = s.split( "#" ); + String className = nameSplit[0]; + String testName = nameSplit[1]; + String parenName = testName + "(" + className + ")"; + addTestToOrders( className, orders, parenName ); } return new Comparator() { - int getRank( String o ) - { - synchronized ( orders ) - { - if ( !orders.containsKey( o ) ) - { - orders.put( o, orders.size() ); - } - return orders.get( o ); - } - } @Override public int compare( String o1, String o2 ) { - int r1 = getRank( o1 ); - int r2 = getRank( o2 ); - if ( r1 < r2 ) + String className1 = o1; + if ( o1.contains( "(" ) ) { - return -1; + String[] nameSplit1 = o1.split( "\\(" ); + className1 = nameSplit1[1].substring( 0, nameSplit1[1].length() - 1 ); + addTestToOrders( className1, orders, o1 ); + } + + String className2 = o2; + if ( o2.contains( "(" ) ) + { + String[] nameSplit2 = o2.split( "\\(" ); + className2 = nameSplit2[1].substring( 0, nameSplit2[1].length() - 1 ); + addTestToOrders( className2, orders, o2 ); + } + + if ( ! className2.equals( className1 ) ) + { + List classOrders = new ArrayList( orders.keySet() ); + if ( classOrders.indexOf( className1 ) < classOrders.indexOf( className2 ) ) + { + return -1; + } + else + { + return 1; + } } else { - return 1; + List testOrders = orders.get( className2 ); + if ( testOrders.indexOf( o1 ) < testOrders.indexOf( o2 ) ) + { + return -1; + } + else + { + return 1; + } } } }; @@ -189,12 +211,21 @@ public int compare( String o1, String o2 ) } } - public String changeTestNameFormatHashToParen( String hashTestName ) + public void addTestToOrders( String className, LinkedHashMap> orders, String parenName ) { - String[] nameSplit = hashTestName.split( "#" ); - String className = nameSplit[0]; - String testName = nameSplit[1]; - return testName + "(" + className + ")"; + List classOrders = orders.get( className ); + if ( classOrders == null ) + { + classOrders = new ArrayList(); + } + if ( ! classOrders.contains( parenName ) ) + { + classOrders.add( parenName ); + } + if ( ! orders.containsKey( className ) ) + { + orders.put( className, classOrders ); + } } private void orderTestClasses( List> testClasses, RunOrder runOrder ) From 38a3dddc072bcd6941ce86e0d2ca14a7928a8025 Mon Sep 17 00:00:00 2001 From: Wing Lam Date: Wed, 3 Mar 2021 02:29:43 +0000 Subject: [PATCH 21/24] Updating readme --- README-bak.md | 74 -------------------------------- README.md | 116 ++++++++++++++++++++++---------------------------- 2 files changed, 52 insertions(+), 138 deletions(-) delete mode 100644 README-bak.md diff --git a/README-bak.md b/README-bak.md deleted file mode 100644 index 83d4ec9829..0000000000 --- a/README-bak.md +++ /dev/null @@ -1,74 +0,0 @@ - - -Maven - -Contributing to [Apache Maven Surefire](https://maven.apache.org/surefire/) -====================== - -[![Maven Central](https://img.shields.io/maven-central/v/org.apache.maven.surefire/surefire.svg?label=Maven%20Central&style=for-the-badge)](https://search.maven.org/#search%7Cgav%7C1%7Cg%3A%22org.apache.maven.plugins%22%20AND%20a%3A%22maven-surefire-plugin%22) -[![Apache License, Version 2.0, January 2004](https://img.shields.io/github/license/apache/maven.svg?label=License&style=for-the-badge)][license] - -[![CI](https://img.shields.io/badge/CI-Jenkins-blue.svg?style=for-the-badge)](https://jenkins-ci.org/) -[![Jenkins Status](https://img.shields.io/jenkins/s/https/builds.apache.org/job/maven-box/job/maven-surefire/job/master.svg?style=for-the-badge)][build] -[![Jenkins tests](https://img.shields.io/jenkins/t/https/builds.apache.org/job/maven-box/job/maven-surefire/job/master.svg?style=for-the-badge)][test-results] -[![Jenkins JaCoCo](https://img.shields.io/jenkins/coverage/jacoco/https/builds.apache.org/job/maven-box/job/maven-surefire/job/master.svg?style=for-the-badge&color=green)](https://builds.apache.org/job/maven-box/job/maven-surefire/job/master/lastBuild/jacoco/) - -[![Actions Status](https://github.com/apache/maven-surefire/workflows/GitHub%20CI/badge.svg?branch=master)](https://github.com/apache/maven-surefire/actions) - -# The Maven Community - -[![slack](https://img.shields.io/badge/slack-18/1138-pink.svg?style=for-the-badge)](https://the-asf.slack.com) -[![forks](https://img.shields.io/github/forks/apache/maven-surefire.svg?style=for-the-badge&label=Fork)](https://github.com/apache/maven-surefire/) - - -# Project Documentation - -[![Maven 3.0 Plugin API](https://img.shields.io/badge/maven%20site-documentation-blue.svg?style=for-the-badge)](https://maven.apache.org/surefire/) - -Usage of [maven-surefire-plugin], [maven-failsafe-plugin], [maven-surefire-report-plugin] - - -# Development Information - -Build the Surefire project using **Maven 3.1.0+** and **JDK 1.8**. - -* In order to run tests for a release check during the Vote, the following memory requirements are needed: - **(on Linux/Unix)** *export MAVEN_OPTS="-server -Xmx512m -XX:MetaspaceSize=128m -XX:MaxMetaspaceSize=384m -XX:+UseG1GC -XX:+UseStringDeduplication -XX:+TieredCompilation -XX:TieredStopAtLevel=1 -XX:SoftRefLRUPolicyMSPerMB=50 -Djava.awt.headless=true -Dhttps.protocols=TLSv1"* - **(on Windows)** *set MAVEN_OPTS="-server -Xmx256m -XX:MetaspaceSize=128m -XX:MaxMetaspaceSize=384m -XX:+UseG1GC -XX:+UseStringDeduplication -XX:+TieredCompilation -XX:TieredStopAtLevel=1 -XX:SoftRefLRUPolicyMSPerMB=50 -Djava.awt.headless=true -Dhttps.protocols=TLSv1"* -* In order to run the build with **JDK 9** **on Windows** (**on Linux/Unix modify system property jdk.home**): - *mvn install site site:stage -P reporting,run-its "-Djdk.home=e:\Program Files\Java\jdk9\"* -* In order to run the build with **JDK 11**: - *mvn install site site:stage -P reporting,run-its "-Djdk.home=e:\Program Files\Java\jdk11\"* - - -### Deploying web site - -See http://maven.apache.org/developers/website/deploy-component-reference-documentation.html - -[![Built with Maven](http://maven.apache.org/images/logos/maven-feather.png)](https://maven.apache.org/surefire/) - - -[license]: https://www.apache.org/licenses/LICENSE-2.0 -[build]: https://builds.apache.org/job/maven-box/job/maven-surefire/job/master/ -[test-results]: https://builds.apache.org/job/maven-box/job/maven-surefire/job/master/lastCompletedBuild/testReport/ -[Join us @ irc://freenode/maven]: https://www.irccloud.com/invite?channel=maven&hostname=irc.freenode.net&port=6697&ssl=1 -[Webchat with us @channel maven]: http://webchat.freenode.net/?channels=%23maven -[JIRA Change Log]: https://issues.apache.org/jira/browse/SUREFIRE/?selectedTab=com.atlassian.jira.jira-projects-plugin:changelog-panel -[maven-surefire-plugin]: https://maven.apache.org/surefire/maven-surefire-plugin/usage.html -[maven-failsafe-plugin]: https://maven.apache.org/surefire/maven-failsafe-plugin/usage.html -[maven-surefire-report-plugin]: https://maven.apache.org/surefire/maven-surefire-report-plugin/usage.html diff --git a/README.md b/README.md index 499a020880..83d4ec9829 100644 --- a/README.md +++ b/README.md @@ -1,86 +1,74 @@ -# Usage + -## Setup +Maven -To use the plugin, please ensure you are on the `test-method-sorting` branch and then perform the following steps. +Contributing to [Apache Maven Surefire](https://maven.apache.org/surefire/) +====================== -1. Run ```mvn install -DskipTests -Drat.skip``` in this directory -2. Copy ```surefire-changing-maven-extension/target/surefire-changing-maven-extension-1.0-SNAPSHOT.jar``` into your Maven installation's ```lib/ext``` directory. This [StackOverflow post](https://stackoverflow.com/a/39479104) may help indicate where your Maven installation is located +[![Maven Central](https://img.shields.io/maven-central/v/org.apache.maven.surefire/surefire.svg?label=Maven%20Central&style=for-the-badge)](https://search.maven.org/#search%7Cgav%7C1%7Cg%3A%22org.apache.maven.plugins%22%20AND%20a%3A%22maven-surefire-plugin%22) +[![Apache License, Version 2.0, January 2004](https://img.shields.io/github/license/apache/maven.svg?label=License&style=for-the-badge)][license] -The copying of the extension helps ensure that any project you run ```mvn test``` on will now use this custom version of Surefire and change certain settings (e.g., reuseForks to false) to prevent issues with fixing the ordering of tests. More information on how to use Maven extensions can be found [here](https://maven.apache.org/examples/maven-3-lifecycle-extensions.html#use-your-extension-in-your-build-s). Note that if you already have ```surefire-changing-maven-extension-1.0-SNAPSHOT.jar``` in your Maven installation's ```lib/ext``` you must first remove the jar before installing again. +[![CI](https://img.shields.io/badge/CI-Jenkins-blue.svg?style=for-the-badge)](https://jenkins-ci.org/) +[![Jenkins Status](https://img.shields.io/jenkins/s/https/builds.apache.org/job/maven-box/job/maven-surefire/job/master.svg?style=for-the-badge)][build] +[![Jenkins tests](https://img.shields.io/jenkins/t/https/builds.apache.org/job/maven-box/job/maven-surefire/job/master.svg?style=for-the-badge)][test-results] +[![Jenkins JaCoCo](https://img.shields.io/jenkins/coverage/jacoco/https/builds.apache.org/job/maven-box/job/maven-surefire/job/master.svg?style=for-the-badge&color=green)](https://builds.apache.org/job/maven-box/job/maven-surefire/job/master/lastBuild/jacoco/) -## Example +[![Actions Status](https://github.com/apache/maven-surefire/workflows/GitHub%20CI/badge.svg?branch=master)](https://github.com/apache/maven-surefire/actions) -``` -mvn test -Dsurefire.methodRunOrder=fixed \ --Dtest=org.apache.dubbo.rpc.protocol.dubbo.DubboLazyConnectTest#testSticky1,\ -org.apache.dubbo.rpc.protocol.dubbo.DubboProtocolTest#testDubboProtocol,\ -org.apache.dubbo.rpc.protocol.dubbo.DubboProtocolTest#testDubboProtocolWithMina\ --pl dubbo-rpc/dubbo-rpc-dubbo -``` +# The Maven Community -By specifying ```-Dsurefire.methodRunOrder=fixed``` Maven test will run the specifed tests in the order that they appear in ```-Dtest```. -Specifically, running the command above will result in the tests running in the following order: +[![slack](https://img.shields.io/badge/slack-18/1138-pink.svg?style=for-the-badge)](https://the-asf.slack.com) +[![forks](https://img.shields.io/github/forks/apache/maven-surefire.svg?style=for-the-badge&label=Fork)](https://github.com/apache/maven-surefire/) -1. org.apache.dubbo.rpc.protocol.dubbo.DubboLazyConnectTest.testSticky1 -2. org.apache.dubbo.rpc.protocol.dubbo.DubboProtocolTest.testDubboProtocol -3. org.apache.dubbo.rpc.protocol.dubbo.DubboProtocolTest.testDubboProtocolWithMina -## Example with file +# Project Documentation -``` -mvn test -Dtest=path_to_file -Dsurefire.methodRunOrder=fixed -pl dubbo-rpc/dubbo-rpc-dubbo -``` +[![Maven 3.0 Plugin API](https://img.shields.io/badge/maven%20site-documentation-blue.svg?style=for-the-badge)](https://maven.apache.org/surefire/) -By specifying ```-Dsurefire.methodRunOrder=fixed``` Maven test will run the specifed tests in the order that they appear in the file ```path_to_file```. Note that the ```path_to_file``` should be an **absolute** path (e.g., ```/home/user/project/test-list```). +Usage of [maven-surefire-plugin], [maven-failsafe-plugin], [maven-surefire-report-plugin] -Assume the content of ```path_to_file``` are the following: -``` -org.apache.dubbo.rpc.protocol.dubbo.DubboLazyConnectTest#testSticky1 -org.apache.dubbo.rpc.protocol.dubbo.DubboProtocolTest#testDubboProtocol -org.apache.dubbo.rpc.protocol.dubbo.DubboProtocolTest#testDubboProtocolWithMina -``` +# Development Information -Then running the Maven command above will result in the tests running in the following order: +Build the Surefire project using **Maven 3.1.0+** and **JDK 1.8**. -1. org.apache.dubbo.rpc.protocol.dubbo.DubboLazyConnectTest.testSticky1 -2. org.apache.dubbo.rpc.protocol.dubbo.DubboProtocolTest.testDubboProtocol -3. org.apache.dubbo.rpc.protocol.dubbo.DubboProtocolTest.testDubboProtocolWithMina +* In order to run tests for a release check during the Vote, the following memory requirements are needed: + **(on Linux/Unix)** *export MAVEN_OPTS="-server -Xmx512m -XX:MetaspaceSize=128m -XX:MaxMetaspaceSize=384m -XX:+UseG1GC -XX:+UseStringDeduplication -XX:+TieredCompilation -XX:TieredStopAtLevel=1 -XX:SoftRefLRUPolicyMSPerMB=50 -Djava.awt.headless=true -Dhttps.protocols=TLSv1"* + **(on Windows)** *set MAVEN_OPTS="-server -Xmx256m -XX:MetaspaceSize=128m -XX:MaxMetaspaceSize=384m -XX:+UseG1GC -XX:+UseStringDeduplication -XX:+TieredCompilation -XX:TieredStopAtLevel=1 -XX:SoftRefLRUPolicyMSPerMB=50 -Djava.awt.headless=true -Dhttps.protocols=TLSv1"* +* In order to run the build with **JDK 9** **on Windows** (**on Linux/Unix modify system property jdk.home**): + *mvn install site site:stage -P reporting,run-its "-Djdk.home=e:\Program Files\Java\jdk9\"* +* In order to run the build with **JDK 11**: + *mvn install site site:stage -P reporting,run-its "-Djdk.home=e:\Program Files\Java\jdk11\"* + +### Deploying web site -## Random with seed +See http://maven.apache.org/developers/website/deploy-component-reference-documentation.html -This specific feature has been merged to [apache/maven-surefire](https://github.com/apache/maven-surefire/pull/309). The other features in this repository will be submitted to Surefire soon. +[![Built with Maven](http://maven.apache.org/images/logos/maven-feather.png)](https://maven.apache.org/surefire/) -Specifically, Surefire will: -1. Output of the random seed used to generate a particular random test order when `-Dsurefire.runOrder.random.seed` and `-Dfailsafe.runOrder.random.seed` are not set. To get the seed, look for the following in the output: -``` -Tests will run in random order. To reproduce ordering use flag -Dsurefire.runOrder.random.seed=28421961536740501 -``` -2. Replay a previously observed random test order by setting `-Dsurefire.runOrder.random.seed` and `-Dfailsafe.runOrder.random.seed` to the seed that observed the random test order - -Note that the seed will control the random of both test classes and test methods if both are set to random. E.g., ```-Dsurefire.runOrder=random -Dsurefire.methodRunOrder=random -Dsurefire.runOrder.random.seed=28421961536740501``` will randomize both test classes and test methods with ```28421961536740501``` as the seed. It is currently not possible to set the seed of the test class and test method to be different. - -Some tests were added to Surefire for this feature. These tests ensure that the setting of the same random seeds do create the same test orders and different random seeds do create different test orders. Note that the inherent randomness of the orders does mean that the tests can be flaky (nondeterministically pass or fail without changes to the code). The current tests have a rate of 0.4% (1/3)^5 of failing. Increasing the number of tests (3) or the number of times to loop (5) would decrease the odds of the tests failing. - -## Caveats - -1. **Test methods from different test classes cannot interleave.** When test methods from various test classes interleave, all test methods from the first time the test class runs will run then as well. E.g., If tests ClassA.testA, ClassB.testA, ClassA.testB are provided, then the run order will be ClassA.testA, ClassA.testB, ClassB.testA. - -2. **FixMethodOrder annotations are ignored.** JUnit 4.11+ provides the annotation [FixMethodOrder](https://junit.org/junit4/javadoc/4.12/org/junit/FixMethodOrder.html) to control the ordering in which tests are run. Such annotations are ignored when this plugin is used. E.g., If tests ClassA.testB, ClassA.testA are provided and FixMethodOrder is set to NAME_ASCENDING, then the run order will still be ClassA.testB, ClassA.testA. - -## TODOs - -The following are features that we would like to have but are yet to be supported. - -1. Have surefire reports save the order in which the test classes are run -2. Allow one to get just the test order list without running tests -3. Reverse mode -4. Have Surefire fix order-dependent tests observed in last run +[license]: https://www.apache.org/licenses/LICENSE-2.0 +[build]: https://builds.apache.org/job/maven-box/job/maven-surefire/job/master/ +[test-results]: https://builds.apache.org/job/maven-box/job/maven-surefire/job/master/lastCompletedBuild/testReport/ +[Join us @ irc://freenode/maven]: https://www.irccloud.com/invite?channel=maven&hostname=irc.freenode.net&port=6697&ssl=1 +[Webchat with us @channel maven]: http://webchat.freenode.net/?channels=%23maven +[JIRA Change Log]: https://issues.apache.org/jira/browse/SUREFIRE/?selectedTab=com.atlassian.jira.jira-projects-plugin:changelog-panel +[maven-surefire-plugin]: https://maven.apache.org/surefire/maven-surefire-plugin/usage.html +[maven-failsafe-plugin]: https://maven.apache.org/surefire/maven-failsafe-plugin/usage.html +[maven-surefire-report-plugin]: https://maven.apache.org/surefire/maven-surefire-report-plugin/usage.html From 28f53b54ed6806403780ce02eba95db099437462 Mon Sep 17 00:00:00 2001 From: Wing Lam Date: Fri, 5 Mar 2021 21:24:37 +0000 Subject: [PATCH 22/24] Cleaning up unnecessary changes. --- maven-failsafe-plugin/pom.xml | 2 +- maven-surefire-common/pom.xml | 2 +- .../plugin/surefire/AbstractSurefireMojo.java | 1 - .../booterclient/BooterSerializer.java | 2 +- ...DeserializerProviderConfigurationTest.java | 2 +- ...rDeserializerStartupConfigurationTest.java | 2 +- maven-surefire-plugin/pom.xml | 2 +- .../maven/plugin/surefire/SurefirePlugin.java | 36 +++++++++---------- maven-surefire-report-plugin/pom.xml | 2 +- pom.xml | 2 +- surefire-api/pom.xml | 2 +- .../api/testset/RunOrderParameters.java | 4 +-- .../api/util/DefaultRunOrderCalculator.java | 32 ++++++++--------- surefire-booter/pom.xml | 2 +- surefire-extensions-api/pom.xml | 2 +- surefire-extensions-spi/pom.xml | 2 +- surefire-grouper/pom.xml | 2 +- surefire-its/pom.xml | 2 +- surefire-logger-api/pom.xml | 2 +- surefire-providers/common-java5/pom.xml | 2 +- surefire-providers/common-junit3/pom.xml | 2 +- surefire-providers/common-junit4/pom.xml | 2 +- surefire-providers/common-junit48/pom.xml | 10 +++--- surefire-providers/pom.xml | 2 +- .../surefire-junit-platform/pom.xml | 2 +- surefire-providers/surefire-junit3/pom.xml | 2 +- surefire-providers/surefire-junit4/pom.xml | 2 +- surefire-providers/surefire-junit47/pom.xml | 2 +- .../surefire/junitcore/JUnitCoreWrapper.java | 2 +- .../surefire/junitcore/Surefire746Test.java | 10 +++--- .../surefire-testng-utils/pom.xml | 2 +- surefire-providers/surefire-testng/pom.xml | 10 +++--- surefire-report-parser/pom.xml | 2 +- surefire-shadefire/pom.xml | 2 +- surefire-shared-utils/pom.xml | 2 +- 35 files changed, 78 insertions(+), 81 deletions(-) diff --git a/maven-failsafe-plugin/pom.xml b/maven-failsafe-plugin/pom.xml index 11ffe9ee05..3089e1fa1a 100644 --- a/maven-failsafe-plugin/pom.xml +++ b/maven-failsafe-plugin/pom.xml @@ -24,7 +24,7 @@ org.apache.maven.surefire surefire - 3.0.0-FLAKYSNAPSHOT + 3.0.0-M6-SNAPSHOT org.apache.maven.plugins diff --git a/maven-surefire-common/pom.xml b/maven-surefire-common/pom.xml index b9f45406e3..f37f072cd5 100644 --- a/maven-surefire-common/pom.xml +++ b/maven-surefire-common/pom.xml @@ -24,7 +24,7 @@ org.apache.maven.surefire surefire - 3.0.0-FLAKYSNAPSHOT + 3.0.0-M6-SNAPSHOT maven-surefire-common diff --git a/maven-surefire-common/src/main/java/org/apache/maven/plugin/surefire/AbstractSurefireMojo.java b/maven-surefire-common/src/main/java/org/apache/maven/plugin/surefire/AbstractSurefireMojo.java index 612da93b40..893b92a90b 100644 --- a/maven-surefire-common/src/main/java/org/apache/maven/plugin/surefire/AbstractSurefireMojo.java +++ b/maven-surefire-common/src/main/java/org/apache/maven/plugin/surefire/AbstractSurefireMojo.java @@ -1135,7 +1135,6 @@ boolean verifyParameters() warnIfWrongShutdownValue(); warnIfNotApplicableSkipAfterFailureCount(); warnIfIllegalTempDir(); - printDefaultSeedIfNecessary(); warnIfForkCountIsZero(); warnIfIllegalFailOnFlakeCount(); printDefaultSeedIfNecessary(); diff --git a/maven-surefire-common/src/main/java/org/apache/maven/plugin/surefire/booterclient/BooterSerializer.java b/maven-surefire-common/src/main/java/org/apache/maven/plugin/surefire/booterclient/BooterSerializer.java index ffc75056b6..174b4bd59e 100644 --- a/maven-surefire-common/src/main/java/org/apache/maven/plugin/surefire/booterclient/BooterSerializer.java +++ b/maven-surefire-common/src/main/java/org/apache/maven/plugin/surefire/booterclient/BooterSerializer.java @@ -54,6 +54,7 @@ import static org.apache.maven.surefire.booter.BooterConstants.INCLUDES_PROPERTY_PREFIX; import static org.apache.maven.surefire.booter.BooterConstants.ISTRIMSTACKTRACE; import static org.apache.maven.surefire.booter.BooterConstants.MAIN_CLI_OPTIONS; +import static org.apache.maven.surefire.booter.BooterConstants.METHOD_RUN_ORDER; import static org.apache.maven.surefire.booter.BooterConstants.PLUGIN_PID; import static org.apache.maven.surefire.booter.BooterConstants.PROCESS_CHECKER; import static org.apache.maven.surefire.booter.BooterConstants.PROVIDER_CONFIGURATION; @@ -63,7 +64,6 @@ import static org.apache.maven.surefire.booter.BooterConstants.RERUN_FAILING_TESTS_COUNT; import static org.apache.maven.surefire.booter.BooterConstants.RUN_ORDER; import static org.apache.maven.surefire.booter.BooterConstants.RUN_STATISTICS_FILE; -import static org.apache.maven.surefire.booter.BooterConstants.METHOD_RUN_ORDER; import static org.apache.maven.surefire.booter.BooterConstants.SHUTDOWN; import static org.apache.maven.surefire.booter.BooterConstants.SOURCE_DIRECTORY; import static org.apache.maven.surefire.booter.BooterConstants.SPECIFIC_TEST_PROPERTY_PREFIX; diff --git a/maven-surefire-common/src/test/java/org/apache/maven/plugin/surefire/booterclient/BooterDeserializerProviderConfigurationTest.java b/maven-surefire-common/src/test/java/org/apache/maven/plugin/surefire/booterclient/BooterDeserializerProviderConfigurationTest.java index 2d7c7f8150..d0b92452df 100644 --- a/maven-surefire-common/src/test/java/org/apache/maven/plugin/surefire/booterclient/BooterDeserializerProviderConfigurationTest.java +++ b/maven-surefire-common/src/test/java/org/apache/maven/plugin/surefire/booterclient/BooterDeserializerProviderConfigurationTest.java @@ -21,6 +21,7 @@ import junit.framework.Assert; import junit.framework.TestCase; +import org.apache.maven.surefire.shared.io.FileUtils; import org.apache.maven.surefire.booter.BooterDeserializer; import org.apache.maven.surefire.booter.ClassLoaderConfiguration; import org.apache.maven.surefire.booter.ClasspathConfiguration; @@ -38,7 +39,6 @@ import org.apache.maven.surefire.api.testset.TestListResolver; import org.apache.maven.surefire.api.testset.TestRequest; import org.apache.maven.surefire.api.util.RunOrder; -import org.apache.maven.surefire.shared.io.FileUtils; import org.apache.maven.surefire.util.MethodRunOrder; import org.junit.After; import org.junit.Before; diff --git a/maven-surefire-common/src/test/java/org/apache/maven/plugin/surefire/booterclient/BooterDeserializerStartupConfigurationTest.java b/maven-surefire-common/src/test/java/org/apache/maven/plugin/surefire/booterclient/BooterDeserializerStartupConfigurationTest.java index e82bb57695..4fb64b54c6 100644 --- a/maven-surefire-common/src/test/java/org/apache/maven/plugin/surefire/booterclient/BooterDeserializerStartupConfigurationTest.java +++ b/maven-surefire-common/src/test/java/org/apache/maven/plugin/surefire/booterclient/BooterDeserializerStartupConfigurationTest.java @@ -31,13 +31,13 @@ import org.apache.maven.surefire.booter.StartupConfiguration; import org.apache.maven.surefire.api.cli.CommandLineOption; import org.apache.maven.surefire.api.report.ReporterConfiguration; +import org.apache.maven.surefire.shared.io.FileUtils; import org.apache.maven.surefire.api.testset.DirectoryScannerParameters; import org.apache.maven.surefire.api.testset.RunOrderParameters; import org.apache.maven.surefire.api.testset.TestArtifactInfo; import org.apache.maven.surefire.api.testset.TestListResolver; import org.apache.maven.surefire.api.testset.TestRequest; import org.apache.maven.surefire.api.util.RunOrder; -import org.apache.maven.surefire.shared.io.FileUtils; import org.apache.maven.surefire.util.MethodRunOrder; import org.junit.After; import org.junit.Before; diff --git a/maven-surefire-plugin/pom.xml b/maven-surefire-plugin/pom.xml index ed5d185fd6..b5c52edbdc 100644 --- a/maven-surefire-plugin/pom.xml +++ b/maven-surefire-plugin/pom.xml @@ -24,7 +24,7 @@ org.apache.maven.surefire surefire - 3.0.0-FLAKYSNAPSHOT + 3.0.0-M6-SNAPSHOT org.apache.maven.plugins diff --git a/maven-surefire-plugin/src/main/java/org/apache/maven/plugin/surefire/SurefirePlugin.java b/maven-surefire-plugin/src/main/java/org/apache/maven/plugin/surefire/SurefirePlugin.java index efdf72eeea..5e547d7b25 100644 --- a/maven-surefire-plugin/src/main/java/org/apache/maven/plugin/surefire/SurefirePlugin.java +++ b/maven-surefire-plugin/src/main/java/org/apache/maven/plugin/surefire/SurefirePlugin.java @@ -20,6 +20,9 @@ */ import java.io.File; +import java.io.IOException; +import java.nio.charset.Charset; +import java.nio.file.Files; import java.util.Arrays; import java.util.Collections; import java.util.List; @@ -33,11 +36,6 @@ import org.apache.maven.surefire.extensions.ForkNodeFactory; import org.apache.maven.surefire.api.suite.RunResult; -import java.nio.file.Files; -import java.nio.charset.Charset; -import java.io.IOException; - - import static org.apache.maven.plugin.surefire.SurefireHelper.reportExecution; /** @@ -640,22 +638,22 @@ public String getTest() { File f = new File( test ); if ( f.exists() && !f.isDirectory ( ) ) + { + try { - try - { - List l = Files.readAllLines( f.toPath(), Charset.defaultCharset( ) ); - StringBuilder sb = new StringBuilder( ); - for ( String s : l ) - { - sb.append( s + "," ); - } - String s = sb.toString( ); - return s.substring( 0 , s.length( ) - 1 ); - } - catch ( IOException e ) - { - } + List l = Files.readAllLines( f.toPath(), Charset.defaultCharset( ) ); + StringBuilder sb = new StringBuilder( ); + for ( String s : l ) + { + sb.append( s + "," ); + } + String s = sb.toString( ); + return s.substring( 0 , s.length( ) - 1 ); } + catch ( IOException e ) + { + } + } return test; } diff --git a/maven-surefire-report-plugin/pom.xml b/maven-surefire-report-plugin/pom.xml index 15e224e9aa..9194a16f8e 100644 --- a/maven-surefire-report-plugin/pom.xml +++ b/maven-surefire-report-plugin/pom.xml @@ -24,7 +24,7 @@ org.apache.maven.surefire surefire - 3.0.0-FLAKYSNAPSHOT + 3.0.0-M6-SNAPSHOT org.apache.maven.plugins diff --git a/pom.xml b/pom.xml index e0888ad444..9e4bb2cc0e 100644 --- a/pom.xml +++ b/pom.xml @@ -28,7 +28,7 @@ org.apache.maven.surefire surefire - 3.0.0-FLAKYSNAPSHOT + 3.0.0-M6-SNAPSHOT pom Apache Maven Surefire diff --git a/surefire-api/pom.xml b/surefire-api/pom.xml index 9e606ad90e..158c88a8cd 100644 --- a/surefire-api/pom.xml +++ b/surefire-api/pom.xml @@ -23,7 +23,7 @@ org.apache.maven.surefire surefire - 3.0.0-FLAKYSNAPSHOT + 3.0.0-M6-SNAPSHOT surefire-api diff --git a/surefire-api/src/main/java/org/apache/maven/surefire/api/testset/RunOrderParameters.java b/surefire-api/src/main/java/org/apache/maven/surefire/api/testset/RunOrderParameters.java index 1cd84bfabe..571f02b91b 100644 --- a/surefire-api/src/main/java/org/apache/maven/surefire/api/testset/RunOrderParameters.java +++ b/surefire-api/src/main/java/org/apache/maven/surefire/api/testset/RunOrderParameters.java @@ -19,11 +19,11 @@ * under the License. */ +import java.io.File; + import org.apache.maven.surefire.api.util.RunOrder; import org.apache.maven.surefire.util.MethodRunOrder; -import java.io.File; - /** * @author Kristian Rosenvold */ diff --git a/surefire-api/src/main/java/org/apache/maven/surefire/api/util/DefaultRunOrderCalculator.java b/surefire-api/src/main/java/org/apache/maven/surefire/api/util/DefaultRunOrderCalculator.java index fc204ef9d0..c2b14ad96e 100644 --- a/surefire-api/src/main/java/org/apache/maven/surefire/api/util/DefaultRunOrderCalculator.java +++ b/surefire-api/src/main/java/org/apache/maven/surefire/api/util/DefaultRunOrderCalculator.java @@ -267,26 +267,26 @@ else if ( sortOrder != null ) private String parseFlakyTestOrder( String s ) { if ( s != null && s != "" ) + { + File f = new File( s ); + if ( f.exists() && !f.isDirectory ( ) ) { - File f = new File( s ); - if ( f.exists() && !f.isDirectory ( ) ) + try + { + List l = Files.readAllLines( f.toPath(), Charset.defaultCharset( ) ); + StringBuilder sb = new StringBuilder( ); + for ( String sd : l ) { - try - { - List l = Files.readAllLines( f.toPath(), Charset.defaultCharset( ) ); - StringBuilder sb = new StringBuilder( ); - for ( String sd : l ) - { - sb.append( sd + "," ); - } - String sd = sb.toString( ); - return sd.substring( 0 , sd.length( ) - 1 ); - } - catch ( IOException e ) - { - } + sb.append( sd + "," ); } + String sd = sb.toString( ); + return sd.substring( 0 , sd.length( ) - 1 ); + } + catch ( IOException e ) + { + } } + } return s; } diff --git a/surefire-booter/pom.xml b/surefire-booter/pom.xml index 0975597b1b..db7e3d02fc 100644 --- a/surefire-booter/pom.xml +++ b/surefire-booter/pom.xml @@ -23,7 +23,7 @@ org.apache.maven.surefire surefire - 3.0.0-FLAKYSNAPSHOT + 3.0.0-M6-SNAPSHOT surefire-booter diff --git a/surefire-extensions-api/pom.xml b/surefire-extensions-api/pom.xml index cafda1e6e1..ee086111df 100644 --- a/surefire-extensions-api/pom.xml +++ b/surefire-extensions-api/pom.xml @@ -24,7 +24,7 @@ org.apache.maven.surefire surefire - 3.0.0-FLAKYSNAPSHOT + 3.0.0-M6-SNAPSHOT surefire-extensions-api diff --git a/surefire-extensions-spi/pom.xml b/surefire-extensions-spi/pom.xml index 70278f22ce..43d0123cf0 100644 --- a/surefire-extensions-spi/pom.xml +++ b/surefire-extensions-spi/pom.xml @@ -23,7 +23,7 @@ org.apache.maven.surefire surefire - 3.0.0-FLAKYSNAPSHOT + 3.0.0-M6-SNAPSHOT surefire-extensions-spi diff --git a/surefire-grouper/pom.xml b/surefire-grouper/pom.xml index 49298d1704..7265a0b9bc 100644 --- a/surefire-grouper/pom.xml +++ b/surefire-grouper/pom.xml @@ -24,7 +24,7 @@ org.apache.maven.surefire surefire - 3.0.0-FLAKYSNAPSHOT + 3.0.0-M6-SNAPSHOT surefire-grouper diff --git a/surefire-its/pom.xml b/surefire-its/pom.xml index 588bbf7283..8275230f33 100644 --- a/surefire-its/pom.xml +++ b/surefire-its/pom.xml @@ -24,7 +24,7 @@ org.apache.maven.surefire surefire - 3.0.0-FLAKYSNAPSHOT + 3.0.0-M6-SNAPSHOT surefire-its diff --git a/surefire-logger-api/pom.xml b/surefire-logger-api/pom.xml index b2e38a6bb1..3bb3e8405a 100644 --- a/surefire-logger-api/pom.xml +++ b/surefire-logger-api/pom.xml @@ -23,7 +23,7 @@ org.apache.maven.surefire surefire - 3.0.0-FLAKYSNAPSHOT + 3.0.0-M6-SNAPSHOT surefire-logger-api diff --git a/surefire-providers/common-java5/pom.xml b/surefire-providers/common-java5/pom.xml index 20e21e75b0..f76659e361 100644 --- a/surefire-providers/common-java5/pom.xml +++ b/surefire-providers/common-java5/pom.xml @@ -23,7 +23,7 @@ org.apache.maven.surefire surefire-providers - 3.0.0-FLAKYSNAPSHOT + 3.0.0-M6-SNAPSHOT common-java5 diff --git a/surefire-providers/common-junit3/pom.xml b/surefire-providers/common-junit3/pom.xml index 49f28aa3a3..265910d474 100644 --- a/surefire-providers/common-junit3/pom.xml +++ b/surefire-providers/common-junit3/pom.xml @@ -23,7 +23,7 @@ org.apache.maven.surefire surefire-providers - 3.0.0-FLAKYSNAPSHOT + 3.0.0-M6-SNAPSHOT common-junit3 diff --git a/surefire-providers/common-junit4/pom.xml b/surefire-providers/common-junit4/pom.xml index 0a32bd5284..54806b4770 100644 --- a/surefire-providers/common-junit4/pom.xml +++ b/surefire-providers/common-junit4/pom.xml @@ -23,7 +23,7 @@ org.apache.maven.surefire surefire-providers - 3.0.0-FLAKYSNAPSHOT + 3.0.0-M6-SNAPSHOT common-junit4 diff --git a/surefire-providers/common-junit48/pom.xml b/surefire-providers/common-junit48/pom.xml index d40b8953af..b9bab4612e 100644 --- a/surefire-providers/common-junit48/pom.xml +++ b/surefire-providers/common-junit48/pom.xml @@ -20,11 +20,11 @@ 4.0.0 - - org.apache.maven.surefire - surefire-providers - 3.0.0-FLAKYSNAPSHOT - + + org.apache.maven.surefire + surefire-providers + 3.0.0-M6-SNAPSHOT + common-junit48 diff --git a/surefire-providers/pom.xml b/surefire-providers/pom.xml index 51c99bb9b4..6c072d38ab 100644 --- a/surefire-providers/pom.xml +++ b/surefire-providers/pom.xml @@ -24,7 +24,7 @@ org.apache.maven.surefire surefire - 3.0.0-FLAKYSNAPSHOT + 3.0.0-M6-SNAPSHOT surefire-providers diff --git a/surefire-providers/surefire-junit-platform/pom.xml b/surefire-providers/surefire-junit-platform/pom.xml index eb2ca72f60..54430dac1a 100644 --- a/surefire-providers/surefire-junit-platform/pom.xml +++ b/surefire-providers/surefire-junit-platform/pom.xml @@ -23,7 +23,7 @@ org.apache.maven.surefire surefire-providers - 3.0.0-FLAKYSNAPSHOT + 3.0.0-M6-SNAPSHOT surefire-junit-platform diff --git a/surefire-providers/surefire-junit3/pom.xml b/surefire-providers/surefire-junit3/pom.xml index 1733a0cb77..3571e64503 100644 --- a/surefire-providers/surefire-junit3/pom.xml +++ b/surefire-providers/surefire-junit3/pom.xml @@ -23,7 +23,7 @@ org.apache.maven.surefire surefire-providers - 3.0.0-FLAKYSNAPSHOT + 3.0.0-M6-SNAPSHOT surefire-junit3 diff --git a/surefire-providers/surefire-junit4/pom.xml b/surefire-providers/surefire-junit4/pom.xml index d6af62d3e4..9670023e87 100644 --- a/surefire-providers/surefire-junit4/pom.xml +++ b/surefire-providers/surefire-junit4/pom.xml @@ -23,7 +23,7 @@ org.apache.maven.surefire surefire-providers - 3.0.0-FLAKYSNAPSHOT + 3.0.0-M6-SNAPSHOT surefire-junit4 diff --git a/surefire-providers/surefire-junit47/pom.xml b/surefire-providers/surefire-junit47/pom.xml index 98846d6035..ddac05f92a 100644 --- a/surefire-providers/surefire-junit47/pom.xml +++ b/surefire-providers/surefire-junit47/pom.xml @@ -23,7 +23,7 @@ org.apache.maven.surefire surefire-providers - 3.0.0-FLAKYSNAPSHOT + 3.0.0-M6-SNAPSHOT surefire-junit47 diff --git a/surefire-providers/surefire-junit47/src/main/java/org/apache/maven/surefire/junitcore/JUnitCoreWrapper.java b/surefire-providers/surefire-junit47/src/main/java/org/apache/maven/surefire/junitcore/JUnitCoreWrapper.java index 4fa497632d..bf45ba5357 100644 --- a/surefire-providers/surefire-junit47/src/main/java/org/apache/maven/surefire/junitcore/JUnitCoreWrapper.java +++ b/surefire-providers/surefire-junit47/src/main/java/org/apache/maven/surefire/junitcore/JUnitCoreWrapper.java @@ -19,12 +19,12 @@ * under the License. */ -import org.apache.maven.surefire.api.util.RunOrderCalculator; import org.apache.maven.surefire.common.junit4.Notifier; import org.apache.maven.surefire.junitcore.pc.ParallelComputer; import org.apache.maven.surefire.junitcore.pc.ParallelComputerBuilder; import org.apache.maven.surefire.api.report.ConsoleStream; import org.apache.maven.surefire.api.testset.TestSetFailedException; +import org.apache.maven.surefire.api.util.RunOrderCalculator; import org.apache.maven.surefire.api.util.TestsToRun; import org.junit.Ignore; import org.junit.runner.Computer; diff --git a/surefire-providers/surefire-junit47/src/test/java/org/apache/maven/surefire/junitcore/Surefire746Test.java b/surefire-providers/surefire-junit47/src/test/java/org/apache/maven/surefire/junitcore/Surefire746Test.java index c61a88bdfa..b5e677de74 100644 --- a/surefire-providers/surefire-junit47/src/test/java/org/apache/maven/surefire/junitcore/Surefire746Test.java +++ b/surefire-providers/surefire-junit47/src/test/java/org/apache/maven/surefire/junitcore/Surefire746Test.java @@ -46,8 +46,6 @@ import org.apache.maven.surefire.api.booter.BaseProviderFactory; import org.apache.maven.surefire.api.booter.ProviderParameterNames; -import org.apache.maven.surefire.api.testset.RunOrderParameters; -import org.apache.maven.surefire.api.util.DefaultRunOrderCalculator; import org.apache.maven.surefire.common.junit4.JUnit4RunListener; import org.apache.maven.surefire.common.junit4.Notifier; import org.apache.maven.surefire.api.report.DefaultDirectConsoleReporter; @@ -55,7 +53,9 @@ import org.apache.maven.surefire.api.report.ReporterFactory; import org.apache.maven.surefire.api.report.RunListener; import org.apache.maven.surefire.api.suite.RunResult; +import org.apache.maven.surefire.api.testset.RunOrderParameters; import org.apache.maven.surefire.api.testset.TestSetFailedException; +import org.apache.maven.surefire.api.util.DefaultRunOrderCalculator; import org.apache.maven.surefire.api.util.TestsToRun; import org.junit.Rule; @@ -139,9 +139,9 @@ public void surefireIsConfused_ByMultipleIgnore_OnClassLevel() throws Exception exception.expect( TestSetFailedException.class ); JUnit4RunListener dummy = new JUnit4RunListener( new MockReporter() ); new JUnitCoreWrapper( new Notifier( dummy, 0 ), jUnitCoreParameters, - new DefaultDirectConsoleReporter( System.out ), - new DefaultRunOrderCalculator( RunOrderParameters.alphabetical(), 1 ) ). - execute( testsToRun, customRunListeners, null ); + new DefaultDirectConsoleReporter( System.out ), + new DefaultRunOrderCalculator( RunOrderParameters.alphabetical(), 1 ) ). + execute( testsToRun, customRunListeners, null ); } finally { diff --git a/surefire-providers/surefire-testng-utils/pom.xml b/surefire-providers/surefire-testng-utils/pom.xml index e815faef83..cd8f0027ae 100644 --- a/surefire-providers/surefire-testng-utils/pom.xml +++ b/surefire-providers/surefire-testng-utils/pom.xml @@ -23,7 +23,7 @@ org.apache.maven.surefire surefire-providers - 3.0.0-FLAKYSNAPSHOT + 3.0.0-M6-SNAPSHOT surefire-testng-utils diff --git a/surefire-providers/surefire-testng/pom.xml b/surefire-providers/surefire-testng/pom.xml index 325ba3247d..b0f14cd4ed 100644 --- a/surefire-providers/surefire-testng/pom.xml +++ b/surefire-providers/surefire-testng/pom.xml @@ -20,11 +20,11 @@ 4.0.0 - - org.apache.maven.surefire - surefire-providers - 3.0.0-FLAKYSNAPSHOT - + + org.apache.maven.surefire + surefire-providers + 3.0.0-M6-SNAPSHOT + surefire-testng diff --git a/surefire-report-parser/pom.xml b/surefire-report-parser/pom.xml index f1ce6b3ee9..b452bd701c 100644 --- a/surefire-report-parser/pom.xml +++ b/surefire-report-parser/pom.xml @@ -24,7 +24,7 @@ org.apache.maven.surefire surefire - 3.0.0-FLAKYSNAPSHOT + 3.0.0-M6-SNAPSHOT surefire-report-parser diff --git a/surefire-shadefire/pom.xml b/surefire-shadefire/pom.xml index 4e2764d840..164e35ccb2 100644 --- a/surefire-shadefire/pom.xml +++ b/surefire-shadefire/pom.xml @@ -23,7 +23,7 @@ org.apache.maven.surefire surefire - 3.0.0-FLAKYSNAPSHOT + 3.0.0-M6-SNAPSHOT surefire-shadefire diff --git a/surefire-shared-utils/pom.xml b/surefire-shared-utils/pom.xml index 15aaea1f0b..c845f5c4d1 100644 --- a/surefire-shared-utils/pom.xml +++ b/surefire-shared-utils/pom.xml @@ -24,7 +24,7 @@ org.apache.maven.surefire surefire - 3.0.0-FLAKYSNAPSHOT + 3.0.0-M6-SNAPSHOT surefire-shared-utils From a0f050e058b20b61929ecc7ce41de04cd6060666 Mon Sep 17 00:00:00 2001 From: Wing Lam Date: Fri, 5 Mar 2021 21:25:21 +0000 Subject: [PATCH 23/24] Removing surefire changing extension --- issues.md | 1 - pom.xml | 1 - surefire-changing-maven-extension/.gitignore | 1 - surefire-changing-maven-extension/pom.xml | 87 ---- .../fun/jvm/surefire/flaky/Configurator.java | 34 -- .../flaky/KPLifecycleParticipant.java | 465 ------------------ 6 files changed, 589 deletions(-) delete mode 100644 issues.md delete mode 100644 surefire-changing-maven-extension/.gitignore delete mode 100644 surefire-changing-maven-extension/pom.xml delete mode 100644 surefire-changing-maven-extension/src/main/java/fun/jvm/surefire/flaky/Configurator.java delete mode 100644 surefire-changing-maven-extension/src/main/java/fun/jvm/surefire/flaky/KPLifecycleParticipant.java diff --git a/issues.md b/issues.md deleted file mode 100644 index 8d987f0433..0000000000 --- a/issues.md +++ /dev/null @@ -1 +0,0 @@ -There are no known issues at this point. diff --git a/pom.xml b/pom.xml index 9e4bb2cc0e..a5ae1cb5bd 100644 --- a/pom.xml +++ b/pom.xml @@ -63,7 +63,6 @@ maven-surefire-report-plugin surefire-its surefire-shared-utils - surefire-changing-maven-extension diff --git a/surefire-changing-maven-extension/.gitignore b/surefire-changing-maven-extension/.gitignore deleted file mode 100644 index eb5a316cbd..0000000000 --- a/surefire-changing-maven-extension/.gitignore +++ /dev/null @@ -1 +0,0 @@ -target diff --git a/surefire-changing-maven-extension/pom.xml b/surefire-changing-maven-extension/pom.xml deleted file mode 100644 index 7281be0943..0000000000 --- a/surefire-changing-maven-extension/pom.xml +++ /dev/null @@ -1,87 +0,0 @@ - - 4.0.0 - - fun.jvm.surefire.flaky - surefire-changing-maven-extension - 1.0-SNAPSHOT - - UTF-8 - - - - MIT License - http://www.opensource.org/licenses/mit-license.php - repo - - - http://jonbell.net/ - - - winglam - Wing Lam - winglam2@illinois.edu - - - jbell - Jonathan Bell - bellj@gmu.edu - - - - - - org.apache.maven - maven-core - 3.6.3 - - - org.apache.maven - maven-embedder - 3.6.3 - - - - org.codehaus.plexus - plexus-component-annotations - 1.5.5 - - - junit - junit - - - - - junit - junit - 4.11 - test - - - - - - maven-compiler-plugin - 3.1 - - 1.7 - 1.7 - - - - - org.codehaus.plexus - plexus-component-metadata - 1.5.5 - - - - generate-metadata - - - - - - - diff --git a/surefire-changing-maven-extension/src/main/java/fun/jvm/surefire/flaky/Configurator.java b/surefire-changing-maven-extension/src/main/java/fun/jvm/surefire/flaky/Configurator.java deleted file mode 100644 index 10b0a6fb26..0000000000 --- a/surefire-changing-maven-extension/src/main/java/fun/jvm/surefire/flaky/Configurator.java +++ /dev/null @@ -1,34 +0,0 @@ -package fun.jvm.surefire.flaky; - -import org.apache.maven.execution.MavenSession; -import org.apache.maven.model.Plugin; -import org.apache.maven.model.PluginExecution; -import org.apache.maven.plugin.MojoFailureException; -import org.apache.maven.project.MavenProject; -import org.codehaus.plexus.util.xml.Xpp3Dom; - -public abstract class Configurator { - - protected MavenSession session; - public Configurator(MavenSession session) throws MojoFailureException { - this.session = session; - } - public abstract void applyConfiguration(MavenProject project, Plugin plugin, PluginExecution pluginExecution) throws MojoFailureException; - - public String getListenerClass(boolean isTestNG) { - return null; - } - - protected Plugin getOrAddPlugin(MavenProject project, String groupId, String artifactID, String version){ - Plugin ret = project.getPlugin(groupId+":"+artifactID); - if(ret == null){ - ret = new Plugin(); - ret.setGroupId(groupId); - ret.setArtifactId(artifactID); - ret.setVersion(version); - project.getBuild().addPlugin(ret); - project.getBuild().getPluginsAsMap().put(groupId+":"+artifactID,ret); - } - return ret; - } -} diff --git a/surefire-changing-maven-extension/src/main/java/fun/jvm/surefire/flaky/KPLifecycleParticipant.java b/surefire-changing-maven-extension/src/main/java/fun/jvm/surefire/flaky/KPLifecycleParticipant.java deleted file mode 100644 index 1dd070121a..0000000000 --- a/surefire-changing-maven-extension/src/main/java/fun/jvm/surefire/flaky/KPLifecycleParticipant.java +++ /dev/null @@ -1,465 +0,0 @@ -package fun.jvm.surefire.flaky; - -import org.apache.maven.AbstractMavenLifecycleParticipant; -import org.apache.maven.MavenExecutionException; -import org.apache.maven.artifact.Artifact; -import org.apache.maven.artifact.repository.ArtifactRepository; -import org.apache.maven.artifact.repository.MavenArtifactRepository; -import org.apache.maven.execution.MavenSession; -import org.apache.maven.model.Dependency; -import org.apache.maven.model.Plugin; -import org.apache.maven.model.PluginExecution; -import org.apache.maven.model.Repository; -import org.apache.maven.plugin.MojoFailureException; -import org.apache.maven.project.MavenProject; -import org.apache.maven.project.artifact.PluginArtifact; -import org.apache.maven.settings.Mirror; -import org.codehaus.plexus.component.annotations.Component; -import org.codehaus.plexus.util.xml.Xpp3Dom; -import org.eclipse.aether.repository.RemoteRepository; - -import java.util.Collections; -import java.util.HashSet; -import java.util.LinkedList; - -@Component(role = AbstractMavenLifecycleParticipant.class, hint = "kp-ext") -public class KPLifecycleParticipant extends AbstractMavenLifecycleParticipant { - public static final String KP_VERSION = "1.0-SNAPSHOT"; - - private static final String ARGLINE_FLAGS = System.getenv("KP_ARGLINE"); - private static final String ADDL_TEST_DEPS = System.getenv("KP_DEPENDENCIES"); - private static final boolean FAIL_FAST = System.getenv("KP_FAIL_ON_FAILED_TEST") != null; - - public static boolean IGNORE_COMPILE_ERRORS = System.getenv("KP_IGNORE_COMPILE_ERRORS") != null; - private static final boolean RECORD_TESTS = System.getenv("KP_RECORD_TESTS") != null; - - private static final boolean DO_JACOCO = true; //System.getenv("KP_JACOCO") != null; - private static final boolean DO_PIT = System.getenv("KP_PIT") != null; - - private static final boolean FORK_PER_TEST = false; //System.getenv("KP_FORK_PER_TEST") != null && !DO_PIT; - static HashSet disabledPlugins = new HashSet(); - - static { - disabledPlugins.add("maven-enforcer-plugin"); - disabledPlugins.add("license-maven-plugin"); - disabledPlugins.add("maven-duplicate-finder-plugin"); - disabledPlugins.add("apache-rat-plugin"); - disabledPlugins.add("cobertura-maven-plugin"); - disabledPlugins.add("jacoco-maven-plugin"); - disabledPlugins.add("maven-dependency-versions-check-plugin"); - disabledPlugins.add("duplicate-finder-maven-plugin"); - } - - LinkedList configurators = new LinkedList<>(); - private MavenSession session; - - private void removeAnnoyingPlugins(MavenProject proj) { - LinkedList newPlugs = new LinkedList<>(); - for (Plugin p : proj.getBuildPlugins()) { - if (disabledPlugins.contains(p.getArtifactId())) { - System.out.println("Warning: KebabPizza disabling incompatible " + p.getGroupId() + ":" + p.getArtifactId() + " from " + proj.getArtifactId()); - } - else{ - newPlugs.add(p); - } - if (System.getProperty("diffcov.mysql") != null) { - //fix for checkstyle in evaluation - if (p.getArtifactId().equals("maven-antrun-plugin") && proj.getName().contains("checkstyle")) { - PluginExecution del = null; - for (PluginExecution pe : p.getExecutions()) { - if (pe.getId().equals("ant-phase-verify")) - del = pe; - } - if (del != null) - p.getExecutions().remove(del); - } - } - } - proj.getBuild().setPlugins(newPlugs); - - //Also, fix terrible junit deps - for (Dependency d : proj.getDependencies()) { - if ("junit".equals(d.getGroupId()) && "junit".equals(d.getArtifactId())) { - if ("4.2".equals(d.getVersion()) || "4.5".equals(d.getVersion()) || "4.4".equals(d.getVersion()) || "4.3".equals(d.getVersion()) || d.getVersion().startsWith("3")) - d.setVersion("4.6"); - } - } - - //also fix broken class files - if(IGNORE_COMPILE_ERRORS) { - for (Plugin p : proj.getBuildPlugins()) { - if (p.getArtifactId().equals("maven-compiler-plugin")) { - p.setVersion("3.8.0"); - for(PluginExecution pe : p.getExecutions()) { - Xpp3Dom config = (Xpp3Dom) pe.getConfiguration(); - if(config == null){ - config = new Xpp3Dom("configuration"); - pe.setConfiguration(config); - } - addOrSetValue("failOnError", "false", config); - addOrSetValue("compilerId", "eclipse", config); - addOrSetValue("fork", "true", config); - } - Dependency dep = new Dependency(); - dep.setArtifactId("plexus-compiler-eclipse"); - dep.setGroupId("org.codehaus.plexus"); - dep.setVersion("2.8.6"); - p.getDependencies().add(dep); - } - } - } - if(proj.getArtifactId().equals("spring-boot-loader-tools")){ - for(Plugin p : proj.getBuildPlugins()){ - if(p.getArtifactId().equals("maven-dependency-plugin")){ - for(PluginExecution pe : p.getExecutions()){ - if(pe.getId().equals("include-layout-jar")){ - pe.setPhase("prepare-package"); - } - } - } - } - } - if(proj.getArtifactId().equals("jboss-as-build")){ - //remove the ant run plugin - for(Plugin p : proj.getBuildPlugins()){ - if(p.getArtifactId().equals("maven-antrun-plugin")){ - p.getExecutions().clear(); - } - } - } - - //Also fix dead pluginrepos - - LinkedList reposToRemove = new LinkedList<>(); - LinkedList newRepos = new LinkedList<>(); - for (RemoteRepository r : proj.getRemotePluginRepositories()) { - if (r.getUrl().startsWith("http://build.eclipse.org")) - reposToRemove.add(r); - else if(r.getUrl().startsWith("http://repo.gradle") || r.getUrl().startsWith("http://repo.spring") || r.getUrl().startsWith("http://maven.spring")){ - reposToRemove.add(r); - RemoteRepository z = new RemoteRepository.Builder(r.getId(),r.getContentType(),r.getUrl().replace("http://","https://")).build(); - newRepos.add(z); - } - } - proj.getRemotePluginRepositories().addAll(newRepos); - if (reposToRemove.size() > 0) - proj.getRemotePluginRepositories().removeAll(reposToRemove); - - reposToRemove = new LinkedList<>(); - newRepos = new LinkedList<>(); - for (RemoteRepository r : proj.getRemoteProjectRepositories()) { - if (r.getUrl().startsWith("http://build.eclipse.org")) - reposToRemove.add(r); - else if(r.getUrl().startsWith("http://repo.gradle") || r.getUrl().startsWith("http://repo.spring") || r.getUrl().startsWith("http://maven.spring")){ - reposToRemove.add(r); - RemoteRepository z = new RemoteRepository.Builder(r.getId(),r.getContentType(),r.getUrl().replace("http://","https://")).build(); - newRepos.add(z); - } - } - proj.getRemoteProjectRepositories().addAll(newRepos); - if (reposToRemove.size() > 0) - proj.getRemoteProjectRepositories().removeAll(reposToRemove); - - - LinkedList areposToRemove = new LinkedList<>(); - LinkedList anewRepos = new LinkedList<>(); - - for (ArtifactRepository r : proj.getRemoteArtifactRepositories()) { - if (r.getUrl().startsWith("http://build.eclipse.org")) - areposToRemove.add(r); - else if (r.getUrl().startsWith("http://repo.gradle") || r.getUrl().startsWith("http://repo.spring") || r.getUrl().startsWith("http://maven.spring")) { - areposToRemove.add(r); - MavenArtifactRepository mar = (MavenArtifactRepository) r; - ArtifactRepository z = new MavenArtifactRepository(mar.getId(), mar.getUrl().replace("http://", "https://"), - mar.getLayout(), mar.getSnapshots(), mar.getReleases()); - anewRepos.add(z); - } - } - proj.getRemoteArtifactRepositories().addAll(anewRepos); - if (areposToRemove.size() > 0) - proj.getRemoteArtifactRepositories().removeAll(areposToRemove); - - } - - public static void addOrSetValue(String key, String value, Xpp3Dom parent){ - if(parent.getChild(key) != null) - parent.getChild(key).setValue(value); - else - { - Xpp3Dom v = new Xpp3Dom(key); - v.setValue(value); - parent.addChild(v); - } - } - public static Xpp3Dom addOrGetKey(String key, Xpp3Dom parent){ - if(parent.getChild(key) != null) - return parent.getChild(key); - else - { - Xpp3Dom v = new Xpp3Dom(key); - parent.addChild(v); - return v; - } - } - public void rewriteSurefireConfiguration(MavenProject project, Plugin p) throws MojoFailureException { - boolean testNG = false; - for (Dependency d : project.getDependencies()) { - if (d.getGroupId().equals("org.testng")) - testNG = true; - } - p.setVersion("3.0.0-FLAKYSNAPSHOT"); -// Dependency d = new Dependency(); -// d.setArtifactId("kp-test-listener"); -// d.setGroupId("edu.gmu.swe.smells"); -// d.setVersion("1.0-SNAPSHOT"); -// d.setScope("test"); -// project.getDependencies().add(d); -// if (ADDL_TEST_DEPS != null) { -// for (String dep : ADDL_TEST_DEPS.split(",")) { -// String[] dat = dep.split(":"); -// d = new Dependency(); -// d.setGroupId(dat[0]); -// d.setArtifactId(dat[1]); -// d.setVersion(dat[2]); -// d.setScope("test"); -// project.getDependencies().add(d); -// } -// } - - p.getDependencies().clear(); - for (PluginExecution pe : p.getExecutions()) { - - Xpp3Dom config = (Xpp3Dom) pe.getConfiguration(); - if (config == null) - config = new Xpp3Dom("configuration"); - rewriteSurefireExecutionConfig(config, testNG); - p.setConfiguration(config); - pe.setConfiguration(config); - for (Configurator c : configurators) - c.applyConfiguration(project, p, pe); - } - - } - - void rewriteSurefireExecutionConfig(Xpp3Dom config, boolean testNG) throws MojoFailureException { - - Xpp3Dom disableXmlReport = config.getChild("disableXmlReport"); - if(disableXmlReport != null) - { - disableXmlReport.setValue("false"); - } - Xpp3Dom argLine = config.getChild("argLine"); - if (argLine == null) { - argLine = new Xpp3Dom("argLine"); - argLine.setValue(""); - config.addChild(argLine); - } - argLine.setValue(argLine.getValue().replace("${surefireArgLine}", "")); - argLine.setValue(argLine.getValue().replace("'${jacocoArgLine}'", "")); - argLine.setValue(argLine.getValue().replace("${jacocoArgLine}", "")); - if (argLine != null && argLine.getValue().equals("${argLine}")) - argLine.setValue("'-XX:OnOutOfMemoryError=kill -9 %p' "); - else if (argLine != null) { - argLine.setValue("'-XX:OnOutOfMemoryError=kill -9 %p' " + argLine.getValue().replace("@{argLine}", "").replace("${argLine}", "").replace("${test.opts.coverage}", "")); - } - - //Now fix if we wanted jacoco or cobertura - if (ARGLINE_FLAGS != null) - argLine.setValue(ARGLINE_FLAGS + " " + argLine.getValue()); - Xpp3Dom parallel = config.getChild("parallel"); - if (parallel != null) - parallel.setValue("none"); - - fixForkMode(config, FORK_PER_TEST); - - - Xpp3Dom properties = config.getChild("properties"); - if (properties == null) { - properties = new Xpp3Dom("properties"); - config.addChild(properties); - } - if (RECORD_TESTS) { - Xpp3Dom prop = new Xpp3Dom("property"); - properties.addChild(prop); - Xpp3Dom propName = new Xpp3Dom("name"); - propName.setValue("listener"); - Xpp3Dom propValue = new Xpp3Dom("value"); - if (testNG) - propValue.setValue("edu.gmu.swe.kp.listener.TestNGExecutionListener"); - else - propValue.setValue("edu.gmu.swe.kp.listener.TestExecutionListener"); - - - prop.addChild(propName); - prop.addChild(propValue); - } - for (Configurator c : configurators) { - String listener = c.getListenerClass(testNG); - if (listener != null) { - Xpp3Dom prop = new Xpp3Dom("property"); - properties.addChild(prop); - Xpp3Dom propName = new Xpp3Dom("name"); - propName.setValue("listener"); - Xpp3Dom propValue = new Xpp3Dom("value"); - propValue.setValue(listener); - prop.addChild(propName); - prop.addChild(propValue); - } - } - - - Xpp3Dom testFailureIgnore = config.getChild("testFailureIgnore"); - if (testFailureIgnore != null) { - testFailureIgnore.setValue((FAIL_FAST ? "false" : "true")); - } else { - testFailureIgnore = new Xpp3Dom("testFailureIgnore"); - testFailureIgnore.setValue((FAIL_FAST ? "false" : "true")); - config.addChild(testFailureIgnore); - } - - Xpp3Dom vars = config.getChild("systemPropertyVariables"); - if (vars == null) { - vars = new Xpp3Dom("systemPropertyVariables"); - config.addChild(vars); - } - } - - void fixForkMode(Xpp3Dom config, boolean forkPerTest) { - Xpp3Dom forkMode = config.getChild("forkMode"); - boolean isSetToFork = false; - if (forkPerTest) { - if (forkMode != null) - forkMode.setValue("perTest"); - else { - Xpp3Dom forkCount = config.getChild("forkCount"); - if (forkCount != null) - forkCount.setValue("1"); - else { - forkCount = new Xpp3Dom("forkCount"); - forkCount.setValue("1"); - config.addChild(forkCount); - } - Xpp3Dom reuseForks = config.getChild("reuseForks"); - if (reuseForks != null) - reuseForks.setValue("false"); - else { - reuseForks = new Xpp3Dom("reuseForks"); - reuseForks.setValue("false"); - config.addChild(reuseForks); - } - } - return; - } else { - //Want to NOT fork per-test - if (forkMode != null) { - forkMode.setValue("once"); - isSetToFork = true; - } - - Xpp3Dom forkCount = config.getChild("forkCount"); - if (forkCount != null && !forkCount.getValue().equals("1")) { - isSetToFork = true; - forkCount.setValue("1"); - } - - Xpp3Dom reuseForks = config.getChild("reuseForks"); - if (reuseForks != null && reuseForks.getValue().equals("false")) { - reuseForks.setValue("true"); - } - - if (!isSetToFork) { - forkCount = new Xpp3Dom("forkCount"); - forkCount.setValue("1"); - config.addChild(forkCount); - } - } - } - - @Override - public void afterSessionStart(MavenSession session) throws MavenExecutionException { - super.afterSessionStart(session); - Mirror springSnaps = new Mirror(); - springSnaps.setMirrorOf("spring-snapshots"); - springSnaps.setUrl("https://repo.spring.io/snapshot"); - session.getRequest().getMirrors().add(springSnaps); - } - - @Override - public void afterProjectsRead(MavenSession session) throws MavenExecutionException { - - this.session = session; -// try { -// if (DO_JACOCO) -// configurators.add(new JacocoConfigurator(session)); -// if (DO_PIT) -// configurators.add(new PitConfigurator(session)); -// } catch (MojoFailureException ex) { -// throw new MavenExecutionException("Unable to create KP configurator", ex); -// } - - //Find out which is the last run of tests - Plugin lastRunOfSurefireOrFailsafe = null; - for (MavenProject p : session.getProjects()) { - for (Plugin o : p.getBuildPlugins()) { - if ((o.getArtifactId().equals("maven-surefire-plugin") || o.getArtifactId().equals("maven-failsafe-plugin")) - && o.getGroupId().equals("org.apache.maven.plugins")) { - lastRunOfSurefireOrFailsafe = o; - } - } - - } - for (MavenProject p : session.getProjects()) { - removeAnnoyingPlugins(p); -// addDependencyPlugin(p); - try { - LinkedList toModify = new LinkedList<>(p.getBuildPlugins()); - for(Artifact pa : p.getPluginArtifacts()){ - if(pa.getArtifactId().equals("maven-surefire-plugin") || pa.getArtifactId().equals("maven-failsafe-plugin")){ - pa.setVersion("3.0.0-FLAKYSNAPSHOT"); - } - } - for (Plugin o : toModify) { - if ((o.getArtifactId().equals("maven-surefire-plugin") && o.getGroupId().equals("org.apache.maven.plugins")) || (o.getArtifactId().equals("maven-failsafe-plugin") && o.getGroupId().equals("org.apache.maven.plugins"))) { - rewriteSurefireConfiguration(p, o); - } - } - } catch (MojoFailureException e) { - e.printStackTrace(); - } - } - - } - - private void fixServers(MavenProject p){ - for(Repository r : p.getRepositories()){ -// if(r.getUrl().startsWith("http://spring")) - } - } - private void addDependencyPlugin(MavenProject p) { - Plugin depPlugin = null; - for(Plugin o : p.getBuildPlugins()){ - if(o.getArtifactId().equals("maven-dependency-plugin") && o.getGroupId().equals("org.apache.maven.plugins")){ - depPlugin = o; - } - } - if(depPlugin == null){ - depPlugin = new Plugin(); - depPlugin.setArtifactId("maven-dependency-plugin"); - depPlugin.setGroupId("org.apache.maven.plugins"); - depPlugin.setVersion("3.1.2"); - p.getBuild().addPlugin(depPlugin); - } - PluginExecution ex = new PluginExecution(); - Xpp3Dom config = new Xpp3Dom("configuration"); - Xpp3Dom outputProperty = new Xpp3Dom("outputProperty"); - outputProperty.setValue("flakyTestClasspath"); - config.addChild(outputProperty); - ex.setConfiguration(config); - ex.setId("kp-cp-build"); - ex.setPhase("generate-sources"); - ex.setGoals(Collections.singletonList("build-classpath")); - depPlugin.addExecution(ex); - - } - -} From f29212d73f56aacf72a96bae29f7bd99f7eee478 Mon Sep 17 00:00:00 2001 From: Wing Lam Date: Fri, 5 Mar 2021 21:28:29 +0000 Subject: [PATCH 24/24] Formatting issues. --- .../maven/surefire/api/testset/RunOrderParameters.java | 1 - surefire-providers/common-junit48/pom.xml | 6 +++--- surefire-providers/surefire-testng/pom.xml | 6 +++--- 3 files changed, 6 insertions(+), 7 deletions(-) diff --git a/surefire-api/src/main/java/org/apache/maven/surefire/api/testset/RunOrderParameters.java b/surefire-api/src/main/java/org/apache/maven/surefire/api/testset/RunOrderParameters.java index 571f02b91b..74138c77f7 100644 --- a/surefire-api/src/main/java/org/apache/maven/surefire/api/testset/RunOrderParameters.java +++ b/surefire-api/src/main/java/org/apache/maven/surefire/api/testset/RunOrderParameters.java @@ -20,7 +20,6 @@ */ import java.io.File; - import org.apache.maven.surefire.api.util.RunOrder; import org.apache.maven.surefire.util.MethodRunOrder; diff --git a/surefire-providers/common-junit48/pom.xml b/surefire-providers/common-junit48/pom.xml index b9bab4612e..2cf6f75da0 100644 --- a/surefire-providers/common-junit48/pom.xml +++ b/surefire-providers/common-junit48/pom.xml @@ -21,9 +21,9 @@ 4.0.0 - org.apache.maven.surefire - surefire-providers - 3.0.0-M6-SNAPSHOT + org.apache.maven.surefire + surefire-providers + 3.0.0-M6-SNAPSHOT common-junit48 diff --git a/surefire-providers/surefire-testng/pom.xml b/surefire-providers/surefire-testng/pom.xml index b0f14cd4ed..c929b4148a 100644 --- a/surefire-providers/surefire-testng/pom.xml +++ b/surefire-providers/surefire-testng/pom.xml @@ -21,9 +21,9 @@ 4.0.0 - org.apache.maven.surefire - surefire-providers - 3.0.0-M6-SNAPSHOT + org.apache.maven.surefire + surefire-providers + 3.0.0-M6-SNAPSHOT surefire-testng