Search CTest output for failure locations (#4418)

CHANGELOG.md

@@ -9,6 +9,7 @@ Features:
 - Add an option to group the default build target dropdown using CMake groups [#3953](https://github.com/microsoft/vscode-cmake-tools/pull/3953) [@itzandroidtab](https://github.com/itzandroidtab)
 - Add `cmake.exclude` setting that allows users to set folders that they want the CMake Tools extension to ignore. [#4112](https://github.com/microsoft/vscode-cmake-tools/issues/4112)
 - Add a command to substitute CMake Cache variables in `launch.json` and `tasks.json`. [#4422](https://github.com/microsoft/vscode-cmake-tools/pull/4422)
+- Add an option to extract details about failing tests from CTest output using regular expressions. [#4420](https://github.com/microsoft/vscode-cmake-tools/issues/4420)
 
 Improvements:
 

package.json

@@ -2271,6 +2271,68 @@
           "markdownDescription": "%cmake-tools.configuration.cmake.ctest.testSuiteDelimiterMaxOccurrence.markdownDescription%",
           "scope": "machine-overridable"
         },
+        "cmake.ctest.failurePatterns": {
+          "oneOf": [
+            {
+              "type": "array",
+              "items": {
+                "oneOf": [
+                  {
+                    "type": "object",
+                    "required": [
+                      "regexp"
+                    ],
+                    "properties": {
+                      "regexp": {
+                        "type": "string",
+                        "description": "%cmake-tools.configuration.cmake.ctest.failurePatterns.regexp%"
+                      },
+                      "file": {
+                        "type": "integer",
+                        "description": "%cmake-tools.configuration.cmake.ctest.failurePatterns.file%",
+                        "default": 1
+                      },
+                      "line": {
+                        "type": "integer",
+                        "description": "%cmake-tools.configuration.cmake.ctest.failurePatterns.line%",
+                        "default": 2
+                      },
+                      "message": {
+                        "type": "integer",
+                        "description": "%cmake-tools.configuration.cmake.ctest.failurePatterns.message%",
+                        "default": 3
+                      },
+                      "actual": {
+                        "type": "integer",
+                        "description": "%cmake-tools.configuration.cmake.ctest.failurePatterns.actual%"
+                      },
+                      "expected": {
+                        "type": "integer",
+                        "description": "%cmake-tools.configuration.cmake.ctest.failurePatterns.expected%"
+                      }
+                    }
+                  },
+                  {
+                    "type": "string"
+                  }
+                ]
+              }
+            },
+            {
+              "type": "string"
+            }
+          ],
+          "default": [
+            {
+              "regexp": "(.*?):(\\d+): *(?:error: *)(.*)"
+            },
+            {
+              "regexp": "(.*?)\\((\\d+)\\): *(?:error: *)(.*)"
+            }
+          ],
+          "markdownDescription": "%cmake-tools.configuration.cmake.ctest.failurePatterns.markdownDescription%",
+          "scope": "machine-overridable"
+        },
         "cmake.ctest.debugLaunchTarget": {
           "type": "string",
           "default": null,

package.nls.json

@@ -126,9 +126,33 @@
         "comment": [
             "Markdown text between `` should not be translated or localized (they represent literal text) and the capitalization, spacing, and punctuation (including the ``) should not be altered."
         ]
-    }, 
+    },
     "cmake-tools.configuration.cmake.ctest.testSuiteDelimiterMaxOccurrence.markdownDescription": {
         "message": "Maximum number of times the delimiter may be used to split the name of the test. `0` means no limit."
+    },
+    "cmake-tools.configuration.cmake.ctest.failurePatterns.markdownDescription": {
+        "message": "Regular expressions for searching CTest output for additional details about failures. All patterns are tried and test failure details from each are collected.\n\nPatterns must have at minimum one capture group to match the name of the `file` where the failure occurred. They can optionally also capture `line`, `message`, `expected`, and `actual`.\n\nFor example, to match a failure line like `path/to/file:47: text of error message`, this pattern matcher could be used:\n```json\n{\n  \"regexp\": \"(.+):(\\\\d+): ?(.*)\",\n  \"file\": 1,\n  \"line\": 2,\n  \"message\": 3\n}\n```\n",
+        "comment": [
+            "Markdown text between `` should not be translated or localized (they represent literal text) and the capitalization, spacing, and punctuation (including the ``) should not be altered."
+        ]
+    },
+    "cmake-tools.configuration.cmake.ctest.failurePatterns.regexp": {
+        "message": "The regular expression to find a failure in the output."
+    },
+    "cmake-tools.configuration.cmake.ctest.failurePatterns.file": {
+        "message": "The match group index of the filename. If omitted 1 is used."
+    },
+    "cmake-tools.configuration.cmake.ctest.failurePatterns.line": {
+        "message": "The match group index of the failure's line. Defaults to 2."
+    },
+    "cmake-tools.configuration.cmake.ctest.failurePatterns.message": {
+        "message": "The match group index of the message. Defaults to 3."
+    },
+    "cmake-tools.configuration.cmake.ctest.failurePatterns.actual": {
+        "message": "The match group index of the actual test output. Defaults to undefined."
+    },
+    "cmake-tools.configuration.cmake.ctest.failurePatterns.expected": {
+        "message": "The match group index of the expected test output. Defaults to undefined."
     },
 	"cmake-tools.configuration.cmake.ctest.debugLaunchTarget.description": "Target name from launch.json to start when debugging a test with CTest. By default and in case of a non-existing target, this will show a picker with all available targets.",
     "cmake-tools.configuration.cmake.parseBuildDiagnostics.description": "Parse compiler output for warnings and errors.",
@@ -269,7 +293,7 @@
         "comment": [
             "The text in parentheses () should not be localized or altered in any way. Also the square brackets and parentheses themselves [] () should not be altered or as well. However, the text inside the square brackets [] should be localized."
         ]
-    }, 
+    },
     "cmake-tools.configuration.views.cmake.outline.description": "Project Outline",
     "cmake-tools.configuration.views.cmake.pinnedCommands.description": "Pinned Commands",
     "cmake-tools.configuration.cmake.additionalKits.description": "Array of paths to custom kit files.",

src/config.ts

@@ -152,6 +152,17 @@ export interface OptionConfig {
     statusBarVisibility: StatusBarOptionVisibility;
 }
 
+export interface FailurePattern {
+    regexp: string;
+    file?: number;
+    line?: number;
+    message?: number;
+    actual?: number;
+    expected?: number;
+}
+
+export type FailurePatternsConfig = (FailurePattern | string)[] | string;
+
 export interface ExtensionConfigurationSettings {
     autoSelectActiveFolder: boolean;
     defaultActiveFolder: string | null;
@@ -174,7 +185,7 @@ export interface ExtensionConfigurationSettings {
     buildToolArgs: string[];
     parallelJobs: number;
     ctestPath: string;
-    ctest: { parallelJobs: number; allowParallelJobs: boolean; testExplorerIntegrationEnabled: boolean; testSuiteDelimiter: string; testSuiteDelimiterMaxOccurrence: number; debugLaunchTarget: string | null };
+    ctest: { parallelJobs: number; allowParallelJobs: boolean; testExplorerIntegrationEnabled: boolean; testSuiteDelimiter: string; testSuiteDelimiterMaxOccurrence: number; failurePatterns: FailurePatternsConfig; debugLaunchTarget: string | null };
     parseBuildDiagnostics: boolean;
     enabledOutputParsers: string[];
     debugConfig: CppDebugConfiguration;
@@ -397,6 +408,9 @@ export class ConfigurationReader implements vscode.Disposable {
     get testSuiteDelimiterMaxOccurrence(): number {
         return this.configData.ctest.testSuiteDelimiterMaxOccurrence;
     }
+    get ctestFailurePatterns(): FailurePatternsConfig {
+        return this.configData.ctest.failurePatterns;
+    }
     get ctestDebugLaunchTarget(): string | null {
         return this.configData.ctest.debugLaunchTarget;
     }
@@ -624,7 +638,7 @@ export class ConfigurationReader implements vscode.Disposable {
         parallelJobs: new vscode.EventEmitter<number>(),
         ctestPath: new vscode.EventEmitter<string>(),
         cpackPath: new vscode.EventEmitter<string>(),
-        ctest: new vscode.EventEmitter<{ parallelJobs: number; allowParallelJobs: boolean; testExplorerIntegrationEnabled: boolean; testSuiteDelimiter: string; testSuiteDelimiterMaxOccurrence: number; debugLaunchTarget: string | null }>(),
+        ctest: new vscode.EventEmitter<{ parallelJobs: number; allowParallelJobs: boolean; testExplorerIntegrationEnabled: boolean; testSuiteDelimiter: string; testSuiteDelimiterMaxOccurrence: number; failurePatterns: FailurePatternsConfig; debugLaunchTarget: string | null }>(),
         parseBuildDiagnostics: new vscode.EventEmitter<boolean>(),
         enabledOutputParsers: new vscode.EventEmitter<string[]>(),
         debugConfig: new vscode.EventEmitter<CppDebugConfiguration>(),

src/ctest.ts

@@ -17,6 +17,7 @@ import { ProjectController } from '@cmt/projectController';
 import { extensionManager } from '@cmt/extension';
 import { CMakeProject } from '@cmt/cmakeProject';
 import { handleCoverageInfoFiles } from '@cmt/coverage';
+import { FailurePattern, FailurePatternsConfig } from '@cmt/config';
 
 nls.config({ messageFormat: nls.MessageFormat.bundle, bundleFormat: nls.BundleFormat.standalone })();
 const localize: nls.LocalizeFunc = nls.loadMessageBundle();
@@ -132,6 +133,61 @@ interface ProjectCoverageConfig {
     coverageInfoFiles: string[];
 }
 
+export function searchOutputForFailures(patterns: FailurePatternsConfig, output: string): vscode.TestMessage[] {
+    output = normalizeLF(output);
+    const messages = [];
+    patterns = Array.isArray(patterns) ? patterns : [patterns];
+    for (let pattern of patterns) {
+        pattern = typeof pattern === 'string' ? { regexp: pattern } : pattern;
+        pattern.file ??= 1;
+        pattern.line ??= 2;
+        pattern.message ??= 3;
+
+        try {
+            for (const match of output.matchAll(RegExp(pattern.regexp, "g"))) {
+                if (pattern.file && match[pattern.file]) {
+                    messages.push(matchToTestMessage(pattern, match));
+                }
+            }
+        } catch (e) {
+            console.error(e);
+        }
+    }
+    return messages;
+}
+
+function matchToTestMessage(pat: FailurePattern, match: RegExpMatchArray): vscode.TestMessage {
+    const file = match[pat.file as number];
+    const line = pat.line ? parseLineMatch(match[pat.line]) : 0;
+    const message = pat.message && match[pat.message]?.trim() || 'Test Failed';
+    const actual = pat.actual ? match[pat.actual] : undefined;
+    const expected = pat.expected ? match[pat.expected] : undefined;
+
+    const testMessage = new vscode.TestMessage(normalizeCRLF(message));
+    testMessage.location = new vscode.Location(
+        vscode.Uri.file(file), new vscode.Position(line, 0)
+    );
+    testMessage.expectedOutput = expected;
+    testMessage.actualOutput = actual;
+    return testMessage;
+}
+
+function normalizeLF(s: string) {
+    return s.replace(/\r\n?/g, '\n');
+}
+
+function normalizeCRLF(s: string) {
+    return s = s.replace(/\r?\n/g, '\r\n');
+}
+
+function parseLineMatch(line: string | null) {
+    const i = parseInt(line || '');
+    if (i) {
+        return i - 1;
+    }
+    return 0;
+}
+
 function parseXmlString<T>(xml: string): Promise<T> {
     return new Promise((resolve, reject) => {
         xml2js.parseString(xml, (err, result) => {
@@ -410,7 +466,13 @@ export class CTestDriver implements vscode.Disposable {
         } else {
             log.info(message.message);
         }
-        run.failed(test, message, duration);
+        const outputMessages = searchOutputForFailures(
+            this.ws.config.ctestFailurePatterns as FailurePatternsConfig,
+            // string cast OK; never passed TestMessage with MarkdownString message
+            message.message as string
+        );
+        const messages = outputMessages.length ? outputMessages : message;
+        run.failed(test, messages, duration);
     }
 
     /**

test/unit-tests/config.test.ts

@@ -31,6 +31,7 @@ function createConfig(conf: Partial<ExtensionConfigurationSettings>): Configurat
             testExplorerIntegrationEnabled: true,
             testSuiteDelimiter: '',
             testSuiteDelimiterMaxOccurrence: 0,
+            failurePatterns: [],
             debugLaunchTarget: null
         },
         parseBuildDiagnostics: true,

test/unit-tests/ctest.test.ts

@@ -1,5 +1,6 @@
-import { readTestResultsFile } from "@cmt/ctest";
+import { readTestResultsFile, searchOutputForFailures } from "@cmt/ctest";
 import { expect, getTestResourceFilePath } from "@test/util";
+import { TestMessage } from "vscode";
 
 suite('CTest test', () => {
     test('Parse XML test results', async () => {
@@ -21,4 +22,61 @@ suite('CTest test', () => {
         const result = await readTestResultsFile(getTestResourceFilePath('TestCMakeCache.txt'));
         expect(result).to.eq(undefined);
     });
+
+    test('Find failure patterns in output', () => {
+        const DEFAULT_MESSAGE = 'Test Failed';
+        const output =
+            '/path/to/file:47: the message\r\n'
+            + 'expected wanted this\r\n'
+            + 'actual got this\r\n'
+            + '/only/required/field::\r\n'
+            + '(42) other message: /path/to/other/file\r\n'
+            + 'actually got one thing\r\n'
+            + 'but wanted another\r\n';
+        const results = searchOutputForFailures([
+            {
+                regexp: /(.*):(\d*): ?(.*)(?:\nexpected (.*))?(?:\nactual (.*))?/.source,
+                expected: 4,
+                actual: 5
+            },
+            {
+                regexp: /\((\d*)\) ([^:]*):\s(.*)\nactually got (.*)\nbut wanted (.*)/.source,
+                file: 3,
+                message: 2,
+                line: 1,
+                actual: 4,
+                expected: 5
+            }
+        ], output);
+        expect(results.length).to.eq(3);
+        const [result1, result2, result3] = results;
+        assertMessageFields(result1, '/path/to/file', 46, 0, 'the message', 'wanted this', 'got this');
+        assertMessageFields(result2, '/only/required/field', 0, 0, DEFAULT_MESSAGE, undefined, undefined);
+        assertMessageFields(result3, '/path/to/other/file', 41, 0, 'other message', 'another', 'one thing');
+
+        const result4 = searchOutputForFailures(/(.*):(\d+):/.source, output)[0];
+        assertMessageFields(result4, '/path/to/file', 46, 0, DEFAULT_MESSAGE, undefined, undefined);
+
+        const results2 = searchOutputForFailures([
+            /\/only(.*)::/.source,
+            /(.*):(\d+): (.*)/.source
+        ], output);
+        expect(results2.length).to.eq(2);
+        const [result5, result6] = results2;
+        assertMessageFields(result5, '/required/field', 0, 0, DEFAULT_MESSAGE, undefined, undefined);
+        assertMessageFields(result6, '/path/to/file', 46, 0, 'the message', undefined, undefined);
+    });
+
+    function assertMessageFields(
+        tm: TestMessage,
+        file: string, line: number, column: number, message: string,
+        expected: string | undefined, actual: string | undefined
+    ): void {
+        expect(tm.message).to.eq(message);
+        expect(tm.location?.uri.path).to.eq(file);
+        expect(tm.location?.range.start.line).to.eq(line);
+        expect(tm.location?.range.start.character).to.eq(column);
+        expect(tm.expectedOutput).to.eq(expected);
+        expect(tm.actualOutput).to.eq(actual);
+    }
 });