Skip to content
This repository was archived by the owner on Mar 27, 2025. It is now read-only.

Commit ff14fcb

Browse files
VahilaVahila
committed
Add tests for Test Result Visualization
1 parent 0432a3f commit ff14fcb

File tree

1 file changed

+340
-0
lines changed

1 file changed

+340
-0
lines changed

‎src/test/java/com/mathworks/ci/systemtests/TestResultVisualizationIT.java

Lines changed: 340 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,340 @@
1+
package com.mathworks.ci.systemtests;
2+
3+
import com.mathworks.ci.MatlabBuildWrapperContent;
4+
import com.mathworks.ci.MatlabInstallationAxis;
5+
import com.mathworks.ci.Message;
6+
import com.mathworks.ci.UseMatlabVersionBuildWrapper;
7+
import com.mathworks.ci.freestyle.RunMatlabBuildBuilder;
8+
import com.mathworks.ci.freestyle.RunMatlabCommandBuilder;
9+
import com.mathworks.ci.freestyle.RunMatlabTestsBuilder;
10+
import hudson.matrix.*;
11+
import hudson.model.FreeStyleBuild;
12+
import hudson.model.FreeStyleProject;
13+
import hudson.model.Result;
14+
import hudson.model.Run;
15+
import hudson.slaves.DumbSlave;
16+
import org.htmlunit.html.*;
17+
import org.jenkinsci.plugins.workflow.cps.CpsFlowDefinition;
18+
import org.jenkinsci.plugins.workflow.job.WorkflowJob;
19+
import org.jenkinsci.plugins.workflow.job.WorkflowRun;
20+
import org.junit.*;
21+
import org.jvnet.hudson.test.ExtractResourceSCM;
22+
import org.jvnet.hudson.test.JenkinsRule;
23+
import org.xml.sax.SAXException;
24+
25+
import java.io.IOException;
26+
import java.net.MalformedURLException;
27+
import java.net.URL;
28+
import java.util.Arrays;
29+
import java.util.List;
30+
import java.util.stream.Collectors;
31+
32+
import static org.junit.Assert.*;
33+
34+
public class TestResultVisualizationIT {
35+
private static JenkinsRule.WebClient jenkinsWebClient;
36+
37+
@Rule
38+
public JenkinsRule jenkins = new JenkinsRule();
39+
40+
41+
@BeforeClass
42+
public static void checkMatlabRoot() {
43+
// Check if the MATLAB_ROOT environment variable is defined
44+
String matlabRoot = System.getenv("MATLAB_ROOT");
45+
Assume.assumeTrue("Not running tests as MATLAB_ROOT environment variable is not defined", matlabRoot != null && !matlabRoot.isEmpty());
46+
}
47+
48+
@Before
49+
public void createJenkinsWebClient(){
50+
jenkinsWebClient = jenkins.createWebClient();
51+
}
52+
53+
// Verify test results are shown in summary
54+
@Test
55+
public void verifyTestResultsSummary() throws Exception {
56+
FreeStyleProject project = jenkins.createFreeStyleProject();
57+
project.setScm(new ExtractResourceSCM(Utilities.getRunMATLABTestsData()));
58+
59+
UseMatlabVersionBuildWrapper buildWrapper = new UseMatlabVersionBuildWrapper();
60+
buildWrapper.setMatlabBuildWrapperContent(new MatlabBuildWrapperContent(Message.getValue("matlab.custom.location"), Utilities.getMatlabRoot()));
61+
project.getBuildWrappersList().add(buildWrapper);
62+
63+
// Run tests through Run Command step
64+
RunMatlabCommandBuilder scriptBuilder = new RunMatlabCommandBuilder();
65+
scriptBuilder.setMatlabCommand("runtests('IncludeSubfolders', true)");
66+
project.getBuildersList().add(scriptBuilder);
67+
68+
// Run tests through Run Build step
69+
RunMatlabBuildBuilder buildtoolBuilder = new RunMatlabBuildBuilder();
70+
buildtoolBuilder.setTasks("test");
71+
project.getBuildersList().add(buildtoolBuilder);
72+
73+
FreeStyleBuild build = project.scheduleBuild2(0).get();
74+
75+
// Verify MATLAB Test Result summary
76+
String[] BuildResultSummary= getTestResultSummaryFromBuildStatusPage(build);
77+
List.of(BuildResultSummary).forEach(summary -> {
78+
assertTrue(summary.contains("Tests run: 4"));
79+
assertTrue(summary.contains("Passed: 1"));
80+
assertTrue(summary.contains("Failed: 3"));
81+
assertTrue(summary.contains("Incomplete: 0"));
82+
assertTrue(summary.contains("Not Run: 0"));
83+
});
84+
}
85+
86+
@Test
87+
public void verifyHyperlinkInSummary() throws Exception {
88+
FreeStyleProject project = jenkins.createFreeStyleProject();
89+
project.setScm(new ExtractResourceSCM(Utilities.getRunMATLABTestsData()));
90+
91+
UseMatlabVersionBuildWrapper buildWrapper = new UseMatlabVersionBuildWrapper();
92+
buildWrapper.setMatlabBuildWrapperContent(new MatlabBuildWrapperContent(Message.getValue("matlab.custom.location"), Utilities.getMatlabRoot()));
93+
project.getBuildWrappersList().add(buildWrapper);
94+
95+
// Run tests through Run Build step
96+
RunMatlabBuildBuilder buildtoolBuilder = new RunMatlabBuildBuilder();
97+
buildtoolBuilder.setTasks("test");
98+
project.getBuildersList().add(buildtoolBuilder);
99+
100+
FreeStyleBuild build = project.scheduleBuild2(0).get();
101+
102+
String hyperlinkInSummary = getHyperlinkFromBuildStatus(build);
103+
104+
String hyperlinkOfTestResultsTab = getTestResultTabLinkFromSidePanel(build);
105+
106+
assertTrue(hyperlinkOfTestResultsTab.contains(hyperlinkInSummary));
107+
}
108+
109+
@Test
110+
public void verifyContentInTestResultsTable() throws Exception{
111+
FreeStyleProject project = jenkins.createFreeStyleProject();
112+
project.setScm(new ExtractResourceSCM(Utilities.getRunMATLABTestsData()));
113+
114+
UseMatlabVersionBuildWrapper buildWrapper = new UseMatlabVersionBuildWrapper();
115+
buildWrapper.setMatlabBuildWrapperContent(new MatlabBuildWrapperContent(Message.getValue("matlab.custom.location"), Utilities.getMatlabRoot()));
116+
project.getBuildWrappersList().add(buildWrapper);
117+
118+
// Run tests through Run Build step
119+
RunMatlabBuildBuilder buildtoolBuilder = new RunMatlabBuildBuilder();
120+
buildtoolBuilder.setTasks("test");
121+
project.getBuildersList().add(buildtoolBuilder);
122+
123+
FreeStyleBuild build = project.scheduleBuild2(0).get();
124+
125+
// Verify button text
126+
String totalTestsButtonText = getButtonText("TOTAL", build);
127+
assertTrue(totalTestsButtonText.contains("4"));
128+
assertTrue(totalTestsButtonText.contains("Total Tests"));
129+
130+
String passedButtonText = getButtonText("PASSED", build);
131+
assertTrue(passedButtonText.contains("1"));
132+
assertTrue(passedButtonText.contains("Passed"));
133+
134+
String failedButtonText = getButtonText("FAILED", build);
135+
assertTrue(failedButtonText.contains("3"));
136+
assertTrue(failedButtonText.contains("Failed"));
137+
138+
String incompleteButtonText = getButtonText("INCOMPLETE", build);
139+
assertTrue(incompleteButtonText.contains("0"));
140+
assertTrue(incompleteButtonText.contains("Incomplete"));
141+
142+
String notRunButtonText = getButtonText("NOT_RUN", build);
143+
assertTrue(notRunButtonText.contains("0"));
144+
assertTrue(notRunButtonText.contains("Not Run"));
145+
146+
// Verify the test filenames shown
147+
String[] expectedTestFiles = {" testMultiply ", " squareTest ", " testSquare ", " testSum "};
148+
String[] testFiles = getTestFilesInTable(build);
149+
assertEquals(testFiles, expectedTestFiles);
150+
151+
// Verify Diagnostics are shown for a failed test.
152+
List<String[]> testsContentForAFile = getTestsInfoForATestFile("testMultiply", build);
153+
assertEquals(testsContentForAFile.size(),1);
154+
assertEquals(testsContentForAFile.get(0)[0].trim(), "testMultiplication"); // 'testMultiply has 'testMultiplication' test
155+
assertNotNull(testsContentForAFile.get(0)[1]); // Diagnostics cant be null as 'testMultiply' fails
156+
assertTrue(testsContentForAFile.get(0)[1].contains("testMultiply/testMultiplication"));
157+
assertNotNull(testsContentForAFile.get(0)[2]); // Duration cannot be null
158+
159+
// Verify Diagnostics is empty for passed test
160+
List<String[]> testsContent= getTestsInfoForATestFile("testSum", build);
161+
assertEquals(testsContent.size(),1);
162+
assertEquals(testsContent.get(0)[0].trim(), "testAddition"); // 'testSum has 'testAddition' test
163+
assertEquals(testsContent.get(0)[1], ""); // No diagnostics as 'testAddition' passes
164+
}
165+
166+
// Verify in matrix project
167+
@Test
168+
public void verifyTestResultsSummaryInMatrixProject() throws Exception {
169+
String matlabRoot = System.getenv("MATLAB_ROOT");
170+
String matlabRoot22b = System.getenv("MATLAB_ROOT_22b");
171+
Assume.assumeTrue("Not running tests as MATLAB_ROOT_22b environment variable is not defined", matlabRoot22b != null && !matlabRoot22b.isEmpty());
172+
173+
Utilities.setMatlabInstallation("MATLAB_PATH_1", matlabRoot, jenkins);
174+
Utilities.setMatlabInstallation("MATLAB_PATH_22b", matlabRoot22b, jenkins);
175+
176+
MatrixProject matrixProject = jenkins.createProject(MatrixProject.class);
177+
MatlabInstallationAxis MATLABAxis = new MatlabInstallationAxis(Arrays.asList("MATLAB_PATH_1", "MATLAB_PATH_22b"));
178+
matrixProject.setAxes(new AxisList(MATLABAxis));
179+
matrixProject.setScm(new ExtractResourceSCM(Utilities.getRunMATLABTestsData()));
180+
181+
// Run tests through Run Build step
182+
RunMatlabBuildBuilder buildtoolBuilder = new RunMatlabBuildBuilder();
183+
buildtoolBuilder.setTasks("test");
184+
matrixProject.getBuildersList().add(buildtoolBuilder);
185+
186+
MatrixBuild build = matrixProject.scheduleBuild2(0).get();
187+
188+
Combination c = new Combination(new AxisList(new MatlabInstallationAxis(Arrays.asList("MATLAB_PATH_1"))), "MATLAB_PATH_1");
189+
MatrixRun run = build.getRun(c);
190+
String[] buildResultSummary= getTestResultSummaryFromBuildStatusPage(run);
191+
List.of(buildResultSummary).forEach(summary -> {
192+
assertTrue(summary.contains("Tests run: 4"));
193+
assertTrue(summary.contains("Passed: 1"));
194+
assertTrue(summary.contains("Failed: 3"));
195+
assertTrue(summary.contains("Incomplete: 0"));
196+
assertTrue(summary.contains("Not Run: 0"));
197+
});
198+
199+
c = new Combination(new AxisList(new MatlabInstallationAxis(Arrays.asList("MATLAB_PATH_22b"))), "MATLAB_PATH_22b");
200+
run = build.getRun(c);
201+
String[] secondBuildResultSummary= getTestResultSummaryFromBuildStatusPage(run);
202+
assertEquals(secondBuildResultSummary.length,0); // As for R2022b the view is not generated
203+
jenkins.assertLogContains(matlabRoot22b,run);
204+
205+
jenkins.assertBuildStatus(Result.FAILURE, run); // As the test task fails
206+
}
207+
208+
// Verify in pipeline project
209+
@Test
210+
public void verifySummaryInDeclarativePipeline() throws Exception {
211+
String script = "pipeline {\n" +
212+
" agent any\n" +
213+
Utilities.getEnvironmentDSL() + "\n" +
214+
" stages{\n" +
215+
" stage('Run MATLAB Command') {\n" +
216+
" steps\n" +
217+
" {\n" +
218+
addTestData() + "\n" +
219+
" runMATLABBuild('test')"+
220+
" }\n" +
221+
" }\n" +
222+
" }\n" +
223+
"}";
224+
WorkflowRun build = getPipelineBuild(script);
225+
226+
// Verify MATLAB Test Result summary
227+
String[] BuildResultSummary= getTestResultSummaryFromBuildStatusPage(build);
228+
List.of(BuildResultSummary).forEach(summary -> {
229+
assertTrue(summary.contains("Tests run: 4"));
230+
assertTrue(summary.contains("Passed: 1"));
231+
assertTrue(summary.contains("Failed: 3"));
232+
assertTrue(summary.contains("Incomplete: 0"));
233+
assertTrue(summary.contains("Not Run: 0"));
234+
});
235+
}
236+
237+
// Verify Master Slave
238+
@Test
239+
public void verifyPipelineOnSlave() throws Exception {
240+
DumbSlave s = jenkins.createOnlineSlave();
241+
String script ="node('!built-in') {" +
242+
Utilities.getEnvironmentScriptedPipeline() + "\n" +
243+
addTestData()+"\n" +
244+
"runMATLABBuild(tasks: 'test') }";
245+
246+
WorkflowRun build = getPipelineBuild(script);
247+
248+
// Verify MATLAB Test Result summary
249+
String[] BuildResultSummary= getTestResultSummaryFromBuildStatusPage(build);
250+
List.of(BuildResultSummary).forEach(summary -> {
251+
assertTrue(summary.contains("Tests run: 4"));
252+
assertTrue(summary.contains("Passed: 1"));
253+
assertTrue(summary.contains("Failed: 3"));
254+
assertTrue(summary.contains("Incomplete: 0"));
255+
assertTrue(summary.contains("Not Run: 0"));
256+
});
257+
258+
jenkins.assertLogNotContains("Running on Jenkins", build);
259+
}
260+
261+
262+
private String[] getTestResultSummaryFromBuildStatusPage(Run<?, ?> build) throws IOException, SAXException {
263+
HtmlPage buildPage = jenkinsWebClient.getPage(build);
264+
List<HtmlElement> summaryElement = buildPage.getByXPath("//*[starts-with(@id, 'matlabTestResults')]");
265+
return summaryElement.stream()
266+
.map(element -> (HtmlElement) element)
267+
.map(HtmlElement::getTextContent)
268+
.toArray(String[]::new);
269+
}
270+
271+
private String getHyperlinkFromBuildStatus(FreeStyleBuild build) throws IOException, SAXException {
272+
HtmlPage buildPage = jenkinsWebClient.getPage(build);
273+
HtmlElement summaryElement = (HtmlElement) buildPage.getByXPath("//*[starts-with(@id, 'matlabTestResults')]").get(0);
274+
HtmlAnchor anchor = summaryElement.getFirstByXPath(".//a");
275+
return anchor.getHrefAttribute();
276+
}
277+
278+
private String getTestResultTabLinkFromSidePanel(FreeStyleBuild build) throws IOException, SAXException {
279+
HtmlPage buildPage = jenkinsWebClient.getPage(build);
280+
HtmlElement jenkinsSidePanelElement = buildPage.getFirstByXPath("//*[@id='side-panel']/div");
281+
HtmlElement buildResultTab = (HtmlElement) jenkinsSidePanelElement.getChildNodes().get(5);
282+
HtmlAnchor href = (HtmlAnchor) buildResultTab.getChildNodes().get(0).getByXPath("//a[span[text()='MATLAB Test Results']]").get(0);
283+
return href.getHrefAttribute();
284+
}
285+
286+
private String getButtonText(String buttonID, FreeStyleBuild build) throws IOException, SAXException {
287+
String pathToTestResultsTab = getHyperlinkFromBuildStatus(build);
288+
HtmlPage testResultsTabPage = jenkinsWebClient.getPage(build, pathToTestResultsTab);
289+
HtmlElement buttonElement = (HtmlElement) testResultsTabPage.getElementById(buttonID);
290+
return buttonElement.getTextContent();
291+
}
292+
293+
private String[] getTestFilesInTable(FreeStyleBuild build) throws IOException, SAXException {
294+
String pathToTestResultsTab = getHyperlinkFromBuildStatus(build);
295+
HtmlPage testResultsTabPage = jenkinsWebClient.getPage(build, pathToTestResultsTab);
296+
297+
HtmlTable tableBodyElement = (HtmlTable) testResultsTabPage.getElementById("testResultsTable");
298+
// Get all rows from the tbody of the table
299+
List<HtmlTableRow> rows = tableBodyElement.getBodies().get(0).getRows();
300+
301+
// Get the test file name from table rows
302+
return rows.stream()
303+
.map(row -> (HtmlElement)row.getFirstByXPath(".//div"))
304+
.map(HtmlElement::getTextContent) // Extract the text content of the <div>
305+
.toArray(String[]::new);
306+
}
307+
308+
private List<String[]> getTestsInfoForATestFile(String testFileName, FreeStyleBuild build) throws IOException, SAXException {
309+
String pathToTestResultsTab = getHyperlinkFromBuildStatus(build);
310+
HtmlPage testResultsTabPage = jenkinsWebClient.getPage(build, pathToTestResultsTab);
311+
312+
HtmlTable mainTable = (HtmlTable) testResultsTabPage.getElementById("testResultsTable");
313+
HtmlTableBody innerTable = mainTable.getRowById(testFileName).getFirstByXPath(".//tbody[@id='matlabTestCasesTableBody']");
314+
// Get all rows from the inner table
315+
List<HtmlTableRow> rows = innerTable.getRows();
316+
// Use streams to extract test names, diagnostics, and durations
317+
List<String[]> testDetails = rows.stream()
318+
.map(row -> {
319+
String testName = row.getCell(0).getTextContent(); // Get test name
320+
String diagnostics = row.getCell(1).getTextContent(); // Get Diagnostics
321+
String duration = row.getCell(2).getTextContent(); // Get Duration
322+
return new String[]{testName, diagnostics, duration};
323+
})
324+
.collect(Collectors.toList());
325+
return testDetails;
326+
}
327+
328+
private String addTestData() throws MalformedURLException {
329+
URL zipFile = Utilities.getRunMATLABTestsData();
330+
String path = " unzip '" + zipFile.getPath() + "'" + "\n";
331+
return path;
332+
}
333+
334+
private WorkflowRun getPipelineBuild(String script) throws Exception{
335+
WorkflowJob project = jenkins.createProject(WorkflowJob.class);
336+
project.setDefinition(new CpsFlowDefinition(script,true));
337+
return project.scheduleBuild2(0).get();
338+
}
339+
340+
}

0 commit comments

Comments
 (0)