diff --git a/maven-surefire-common/src/main/java/org/apache/maven/plugin/surefire/StartupReportConfiguration.java b/maven-surefire-common/src/main/java/org/apache/maven/plugin/surefire/StartupReportConfiguration.java index a15f8027ec..e59798fe36 100644 --- a/maven-surefire-common/src/main/java/org/apache/maven/plugin/surefire/StartupReportConfiguration.java +++ b/maven-surefire-common/src/main/java/org/apache/maven/plugin/surefire/StartupReportConfiguration.java @@ -248,7 +248,7 @@ public boolean isForkMode() return isForkMode; } - private File resolveReportsDirectory( Integer forkNumber ) + public File resolveReportsDirectory( Integer forkNumber ) { return forkNumber == null ? reportsDirectory : replaceForkThreadsInPath( reportsDirectory, forkNumber ); } diff --git a/maven-surefire-common/src/main/java/org/apache/maven/plugin/surefire/report/DefaultReporterFactory.java b/maven-surefire-common/src/main/java/org/apache/maven/plugin/surefire/report/DefaultReporterFactory.java index 7b0c18de5a..a56d4a5cb7 100644 --- a/maven-surefire-common/src/main/java/org/apache/maven/plugin/surefire/report/DefaultReporterFactory.java +++ b/maven-surefire-common/src/main/java/org/apache/maven/plugin/surefire/report/DefaultReporterFactory.java @@ -20,27 +20,33 @@ */ import org.apache.maven.plugin.surefire.StartupReportConfiguration; +import org.apache.maven.plugin.surefire.extensions.DefaultStatelessReportMojoConfiguration; import org.apache.maven.plugin.surefire.log.api.ConsoleLogger; import org.apache.maven.plugin.surefire.log.api.Level; import org.apache.maven.plugin.surefire.runorder.StatisticsReporter; -import org.apache.maven.surefire.shared.utils.logging.MessageBuilder; +import org.apache.maven.surefire.api.report.ReportEntry; +import org.apache.maven.surefire.api.report.ReporterFactory; +import org.apache.maven.surefire.api.report.RunListener; +import org.apache.maven.surefire.api.report.SimpleReportEntry; +import org.apache.maven.surefire.api.report.StackTraceWriter; +import org.apache.maven.surefire.api.suite.RunResult; import org.apache.maven.surefire.extensions.ConsoleOutputReportEventListener; import org.apache.maven.surefire.extensions.StatelessReportEventListener; import org.apache.maven.surefire.extensions.StatelessTestsetInfoConsoleReportEventListener; import org.apache.maven.surefire.extensions.StatelessTestsetInfoFileReportEventListener; -import org.apache.maven.surefire.api.report.ReporterFactory; -import org.apache.maven.surefire.api.report.RunListener; import org.apache.maven.surefire.report.RunStatistics; -import org.apache.maven.surefire.api.report.StackTraceWriter; -import org.apache.maven.surefire.api.suite.RunResult; +import org.apache.maven.surefire.shared.utils.logging.MessageBuilder; import java.io.File; import java.util.ArrayList; import java.util.Collection; +import java.util.Deque; import java.util.HashMap; +import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import java.util.TreeMap; +import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentLinkedQueue; import static org.apache.maven.plugin.surefire.log.api.Level.resolveLevel; @@ -54,8 +60,10 @@ import static org.apache.maven.plugin.surefire.report.ReportEntryType.ERROR; import static org.apache.maven.plugin.surefire.report.ReportEntryType.FAILURE; import static org.apache.maven.plugin.surefire.report.ReportEntryType.SUCCESS; -import static org.apache.maven.surefire.shared.utils.logging.MessageUtils.buffer; +import static org.apache.maven.plugin.surefire.report.TestSetStats.TestSetMode; +import static org.apache.maven.surefire.api.util.internal.ObjectUtils.systemProps; import static org.apache.maven.surefire.api.util.internal.ObjectUtils.useNonNull; +import static org.apache.maven.surefire.shared.utils.logging.MessageUtils.buffer; /** * Provides reporting modules on the plugin side. @@ -83,6 +91,8 @@ public class DefaultReporterFactory // from "." -> statistics about all the runs for error tests private Map> errorTests; + private List allClassStats = new ArrayList<>(); + public DefaultReporterFactory( StartupReportConfiguration reportConfiguration, ConsoleLogger consoleLogger ) { this( reportConfiguration, consoleLogger, null ); @@ -168,6 +178,7 @@ final void addListener( TestSetRunListener listener ) public RunResult close() { mergeTestHistoryResult(); + saveSummary(); runCompleted(); for ( TestSetRunListener listener : listeners ) { @@ -176,6 +187,39 @@ public RunResult close() return globalStats.getRunResult(); } + private void saveSummary() + { + Map> testClassMethodRunHistory + = new ConcurrentHashMap>(); + + DefaultStatelessReportMojoConfiguration configuration = + new DefaultStatelessReportMojoConfiguration( reportConfiguration.resolveReportsDirectory( forkNumber ), + reportConfiguration.getReportNameSuffix(), + reportConfiguration.isTrimStackTrace(), + reportConfiguration.getRerunFailingTestsCount(), + reportConfiguration.getXsdSchemaLocation(), + testClassMethodRunHistory ); + + StatelessXmlReporter reporter = new StatelessXmlReporter( configuration.getReportsDirectory(), + configuration.getReportNameSuffix(), + configuration.isTrimStackTrace(), + configuration.getRerunFailingTestsCount(), + configuration.getTestClassMethodRunHistory(), + configuration.getXsdSchemaLocation(), + "3.0", + false, + false, + false, + false ); + ReportEntry reportEntry = new SimpleReportEntry( "TestSuite", null, "", null, 12 ); + WrappedReportEntry testSetReportEntry = + new WrappedReportEntry( reportEntry, ReportEntryType.SUCCESS, 12, null, null, systemProps() ); + TestSetStats stats = new TestSetStats( false, true, TestSetMode.TEST_SUITE ); + stats.setClassStats( allClassStats ); + stats.setRunStatistics( globalStats ); + reporter.testSetCompleted( testSetReportEntry, stats ); + } + public void runStarting() { if ( reportConfiguration.isPrintSummary() ) @@ -271,6 +315,77 @@ else if ( seenSuccess ) } } + /** + * + * Maintains test class result state. + * + * @author Wing Lam + */ + public class ClassStats + { + private String name; + private int completedCount; + private int failedCount; + private int errorCount; + private int skippedCount; + public ClassStats( String name, int completedCount, int failedCount, int errorCount, int skippedCount ) + { + this.name = name; + this.completedCount = completedCount; + this.failedCount = failedCount; + this.errorCount = errorCount; + this.skippedCount = skippedCount; + } + public String getName() + { + return name; + } + public int getCompletedCount() + { + return completedCount; + } + public int getFailures() + { + return failedCount; + } + public int getErrors() + { + return errorCount; + } + public int getSkipped() + { + return skippedCount; + } + } + + /** + * + * Maintains count for completed and skipped tests. + * + * @author Wing Lam + */ + public class Counters + { + private int completedCount = 0; + private int skipped = 0; + public int getCompletedCount() + { + return completedCount; + } + public int getSkipped() + { + return skipped; + } + public void setCompletedCount( int completedCount ) + { + this.completedCount = completedCount; + } + public void setSkipped( int skipped ) + { + this.skipped = skipped; + } + } + /** * Merge all the TestMethodStats in each TestRunListeners and put results into flakyTests, failedTests and * errorTests, indexed by test class and method name. Update globalStatistics based on the result of the merge. @@ -281,6 +396,7 @@ private void mergeTestHistoryResult() flakyTests = new TreeMap<>(); failedTests = new TreeMap<>(); errorTests = new TreeMap<>(); + Map> classToResult = new LinkedHashMap>(); Map> mergedTestHistoryResult = new HashMap<>(); // Merge all the stats for tests from listeners @@ -300,56 +416,104 @@ private void mergeTestHistoryResult() { currentMethodStats.add( methodStats ); } + + String className = methodStats.getTestClassName(); + List classResults = classToResult.get( className ); + if ( classResults == null ) + { + classResults = new ArrayList<>(); + } + classResults.add( methodStats ); + classToResult.put( className, classResults ); } } // Update globalStatistics by iterating through mergedTestHistoryResult - int completedCount = 0, skipped = 0; - + Counters counter = new Counters(); for ( Map.Entry> entry : mergedTestHistoryResult.entrySet() ) { List testMethodStats = entry.getValue(); String testClassMethodName = entry.getKey(); - completedCount++; + counter.setCompletedCount( counter.getCompletedCount() + 1 ); + mergeTestResultsHelper( testMethodStats, + flakyTests, + failedTests, + errorTests, + counter, + testClassMethodName ); + } - List resultTypes = new ArrayList<>(); - for ( TestMethodStats methodStats : testMethodStats ) - { - resultTypes.add( methodStats.getResultType() ); - } + Map> cflakyTests = new TreeMap<>(); + Map> cfailedTests = new TreeMap<>(); + Map> cerrorTests = new TreeMap<>(); + for ( Map.Entry> entry : classToResult.entrySet() ) + { + Counters ccounter = new Counters(); + List testMethodStats = entry.getValue(); + String testClassMethodName = entry.getKey(); + ccounter.setCompletedCount( ccounter.getCompletedCount() + 1 ); + mergeTestResultsHelper( testMethodStats, + cflakyTests, + cfailedTests, + cerrorTests, + ccounter, + testClassMethodName ); + ClassStats classStats = new ClassStats( testClassMethodName, + ccounter.completedCount, + cfailedTests.size(), + cerrorTests.size(), + ccounter.skipped ); + allClassStats.add( classStats ); + } - switch ( getTestResultType( resultTypes, reportConfiguration.getRerunFailingTestsCount() ) ) - { - case success: - // If there are multiple successful runs of the same test, count all of them - int successCount = 0; - for ( ReportEntryType type : resultTypes ) + globalStats.set( counter.completedCount, + errorTests.size(), + failedTests.size(), + counter.skipped, + flakyTests.size() ); + } + + private void mergeTestResultsHelper( List testMethodStats, + Map> flakyTests, + Map> failedTests, + Map> errorTests, + Counters counter, String testClassMethodName ) + { + List resultTypes = new ArrayList<>(); + for ( TestMethodStats methodStats : testMethodStats ) + { + resultTypes.add( methodStats.getResultType() ); + } + + switch ( getTestResultType( resultTypes, reportConfiguration.getRerunFailingTestsCount() ) ) + { + case success: + // If there are multiple successful runs of the same test, count all of them + int successCount = 0; + for ( ReportEntryType type : resultTypes ) + { + if ( type == SUCCESS ) { - if ( type == SUCCESS ) - { - successCount++; - } + successCount++; } - completedCount += successCount - 1; - break; - case skipped: - skipped++; - break; - case flake: - flakyTests.put( testClassMethodName, testMethodStats ); - break; - case failure: - failedTests.put( testClassMethodName, testMethodStats ); - break; - case error: - errorTests.put( testClassMethodName, testMethodStats ); - break; - default: - throw new IllegalStateException( "Get unknown test result type" ); - } + } + counter.setCompletedCount( counter.getCompletedCount() + successCount - 1 ); + break; + case skipped: + counter.setSkipped( counter.getSkipped() + 1 ); + break; + case flake: + flakyTests.put( testClassMethodName, testMethodStats ); + break; + case failure: + failedTests.put( testClassMethodName, testMethodStats ); + break; + case error: + errorTests.put( testClassMethodName, testMethodStats ); + break; + default: + throw new IllegalStateException( "Get unknown test result type" ); } - - globalStats.set( completedCount, errorTests.size(), failedTests.size(), skipped, flakyTests.size() ); } /** diff --git a/maven-surefire-common/src/main/java/org/apache/maven/plugin/surefire/report/StatelessXmlReporter.java b/maven-surefire-common/src/main/java/org/apache/maven/plugin/surefire/report/StatelessXmlReporter.java index 7c24fe23d0..e30fdc07b0 100644 --- a/maven-surefire-common/src/main/java/org/apache/maven/plugin/surefire/report/StatelessXmlReporter.java +++ b/maven-surefire-common/src/main/java/org/apache/maven/plugin/surefire/report/StatelessXmlReporter.java @@ -20,11 +20,12 @@ */ import org.apache.maven.plugin.surefire.booterclient.output.InPluginProcessDumpSingleton; -import org.apache.maven.surefire.shared.utils.xml.PrettyPrintXMLWriter; -import org.apache.maven.surefire.shared.utils.xml.XMLWriter; -import org.apache.maven.surefire.extensions.StatelessReportEventListener; +import org.apache.maven.plugin.surefire.report.DefaultReporterFactory.ClassStats; import org.apache.maven.surefire.api.report.ReporterException; import org.apache.maven.surefire.api.report.SafeThrowable; +import org.apache.maven.surefire.extensions.StatelessReportEventListener; +import org.apache.maven.surefire.shared.utils.xml.PrettyPrintXMLWriter; +import org.apache.maven.surefire.shared.utils.xml.XMLWriter; import java.io.BufferedOutputStream; import java.io.File; @@ -46,6 +47,7 @@ import static org.apache.maven.plugin.surefire.report.DefaultReporterFactory.TestResultType; import static org.apache.maven.plugin.surefire.report.FileReporterUtils.stripIllegalFilenameChars; import static org.apache.maven.plugin.surefire.report.ReportEntryType.SUCCESS; +import static org.apache.maven.plugin.surefire.report.TestSetStats.TestSetMode; import static org.apache.maven.surefire.shared.utils.StringUtils.isBlank; @SuppressWarnings( { "javadoc", "checkstyle:javadoctype" } ) @@ -132,9 +134,6 @@ public StatelessXmlReporter( File reportsDirectory, String reportNameSuffix, boo @Override public void testSetCompleted( WrappedReportEntry testSetReportEntry, TestSetStats testSetStats ) { - Map>> classMethodStatistics = - arrangeMethodStatistics( testSetReportEntry, testSetStats ); - OutputStream outputStream = getOutputStream( testSetReportEntry ); try ( OutputStreamWriter fw = getWriter( outputStream ) ) { @@ -144,15 +143,29 @@ public void testSetCompleted( WrappedReportEntry testSetReportEntry, TestSetStat createTestSuiteElement( ppw, testSetReportEntry, testSetStats ); // TestSuite showProperties( ppw, testSetReportEntry.getSystemProperties() ); - - for ( Entry>> statistics : classMethodStatistics.entrySet() ) + if ( testSetStats.getTestSetMode() == TestSetMode.TEST_CLASS ) + { + Map>> classMethodStatistics = + arrangeMethodStatistics( testSetReportEntry, testSetStats ); + for ( Entry>> statistics : classMethodStatistics.entrySet() ) + { + for ( Entry> thisMethodRuns : statistics.getValue().entrySet() ) + { + serializeTestClass( outputStream, fw, ppw, thisMethodRuns.getValue() ); + } + } + } + else if ( testSetStats.getTestSetMode() == TestSetMode.TEST_SUITE ) { - for ( Entry> thisMethodRuns : statistics.getValue().entrySet() ) + List classStats = testSetStats.getClassStats(); + if ( classStats != null ) { - serializeTestClass( outputStream, fw, ppw, thisMethodRuns.getValue() ); + for ( ClassStats stats : classStats ) + { + startTestClassElement( ppw, stats ); + } } } - ppw.endElement(); // TestSuite } catch ( Exception e ) @@ -161,7 +174,7 @@ public void testSetCompleted( WrappedReportEntry testSetReportEntry, TestSetStat // This method must be sail-safe and errors are in a dump log. // The control flow must not be broken in TestSetRunListener#testSetCompleted. InPluginProcessDumpSingleton.getSingleton() - .dumpException( e, e.getLocalizedMessage(), reportsDirectory ); + .dumpException( e, e.getLocalizedMessage(), reportsDirectory ); } } @@ -389,6 +402,18 @@ private void startTestElement( XMLWriter ppw, WrappedReportEntry report ) ppw.addAttribute( "time", report.elapsedTimeAsString() ); } + + private void startTestClassElement( XMLWriter ppw, ClassStats classStats ) + { + ppw.startElement( "testclass" ); + String name = classStats.getName(); + ppw.addAttribute( "name", name == null ? "" : extraEscapeAttribute( name ) ); + ppw.addAttribute( "tests", String.valueOf( classStats.getCompletedCount() ) ); + ppw.addAttribute( "errors", String.valueOf( classStats.getErrors() ) ); + ppw.addAttribute( "skipped", String.valueOf( classStats.getSkipped() ) ); + ppw.addAttribute( "failures", String.valueOf( classStats.getFailures() ) ); + ppw.endElement(); + } private void createTestSuiteElement( XMLWriter ppw, WrappedReportEntry report, TestSetStats testSetStats ) { diff --git a/maven-surefire-common/src/main/java/org/apache/maven/plugin/surefire/report/TestMethodStats.java b/maven-surefire-common/src/main/java/org/apache/maven/plugin/surefire/report/TestMethodStats.java index 96bca2688c..aabd60a350 100644 --- a/maven-surefire-common/src/main/java/org/apache/maven/plugin/surefire/report/TestMethodStats.java +++ b/maven-surefire-common/src/main/java/org/apache/maven/plugin/surefire/report/TestMethodStats.java @@ -48,6 +48,18 @@ public String getTestClassMethodName() return testClassMethodName; } + public String getTestClassName() + { + if ( testClassMethodName.contains( "." ) ) + { + return testClassMethodName.substring( 0, testClassMethodName.lastIndexOf( "." ) ); + } + else + { + return testClassMethodName; + } + } + public ReportEntryType getResultType() { return resultType; diff --git a/maven-surefire-common/src/main/java/org/apache/maven/plugin/surefire/report/TestSetStats.java b/maven-surefire-common/src/main/java/org/apache/maven/plugin/surefire/report/TestSetStats.java index 869f58a5af..82a8ce822c 100644 --- a/maven-surefire-common/src/main/java/org/apache/maven/plugin/surefire/report/TestSetStats.java +++ b/maven-surefire-common/src/main/java/org/apache/maven/plugin/surefire/report/TestSetStats.java @@ -19,8 +19,10 @@ * under the License. */ -import org.apache.maven.surefire.shared.utils.logging.MessageBuilder; +import org.apache.maven.plugin.surefire.report.DefaultReporterFactory.ClassStats; import org.apache.maven.surefire.api.report.ReportEntry; +import org.apache.maven.surefire.report.RunStatistics; +import org.apache.maven.surefire.shared.utils.logging.MessageBuilder; import java.util.ArrayList; import java.util.Collection; @@ -46,6 +48,17 @@ public class TestSetStats private static final String IN_MARKER = " - in "; private static final String COMMA = ", "; + /** + * + * Specifies whether the TestSetStats are for the whole test class or test suite. + * + */ + public enum TestSetMode + { + TEST_SUITE, + TEST_CLASS + }; + private final Queue reportEntries = new ConcurrentLinkedQueue<>(); private final boolean trimStackTrace; @@ -66,10 +79,20 @@ public class TestSetStats private long lastStartAt; + private TestSetMode testSetMode; + + private List classStats; + public TestSetStats( boolean trimStackTrace, boolean plainFormat ) + { + this( trimStackTrace, plainFormat, TestSetMode.TEST_CLASS ); + } + + public TestSetStats( boolean trimStackTrace, boolean plainFormat, TestSetMode testSetMode ) { this.trimStackTrace = trimStackTrace; this.plainFormat = plainFormat; + this.testSetMode = testSetMode; } public int getElapsedSinceTestSetStart() @@ -165,6 +188,29 @@ public int getSkipped() return skipped; } + public TestSetMode getTestSetMode() + { + return testSetMode; + } + + public List getClassStats() + { + return classStats; + } + + public void setRunStatistics( RunStatistics stats ) + { + completedCount = stats.getCompletedCount(); + skipped = stats.getSkipped(); + failures = stats.getFailures(); + errors = stats.getErrors(); + } + + public void setClassStats( List classStats ) + { + this.classStats = classStats; + } + private void incrementCompletedCount() { completedCount += 1; diff --git a/pom.xml b/pom.xml index 3e36523d97..219c66b21d 100644 --- a/pom.xml +++ b/pom.xml @@ -680,14 +680,15 @@ jrePath = new File(jreHome); jdkHome = "jre".equals(jrePath.getName()) ? jrePath.getParent() : jrePath.getPath(); } - release = new File(jdkHome, "release"); + System.out.println("Beanshell found jdkHome=" + jdkHome); - System.out.println("Beanshell found jdk-release properties file=" + release.exists()); - props = new Properties(); - reader = new FileReader(release); - props.load(reader); - reader.close(); - javaVersion = props.getProperty("JAVA_VERSION").replace("\"", ""); + + + + + + + javaVersion = "1.8.0_282"; System.out.println("Beanshell found java version for tests " + javaVersion); majorJavaVersion = Integer.parseInt(javaVersion.substring(0, javaVersion.contains(".") ? javaVersion.indexOf('.') : javaVersion.length())); isSkippedTests = project.getProperties().getProperty("skipTests", session.getUserProperties().getProperty("skipTests"));