Merge pull request #491 from boxine/junit-output

Add option to output test results as JUnit report

Author: flbraun

README.md

@@ -8,7 +8,7 @@ Browser tests using [puppeteer](https://pptr.dev/) benefit from special support
 
 Depending on the environment (you can set up configurations to run the same tests against dev, stage, prod etc.), tests can be skipped, or marked as _expected to fail_ for test driven development where you write tests first before fixing a bug or implementing a feature.
 A locking system prevents two tests or the same tests on two different machines from accessing a shared resource, e.g. a test account.
-You can review test results in a PDF report.
+You can review test results in a PDF report or create a JUnit report file to be consumed by CI systems.
 
 ## Installation
 
@@ -391,6 +391,9 @@ The keys are up to you; for example you probably want to have a main entry point
 -J, --json            Write tests results as a JSON file.
 --json-file FILE.json
                       JSON file to write to. Defaults to results.json .
+--junit               Write tests results as a JUnit XML file.
+--junit-file FILE.xml
+                      JUnit XML file to write to. Defaults to results.xml .
 -H, --html            Write test results as an HTML file.
 --html-file FILE.html
                       HTML file to write a report to. Defaults to results.html .

src/config.js

@@ -91,6 +91,7 @@ function computeConcurrency(spec, { cpuCount = undefined } = {}) {
 function parseArgs(options, raw_args) {
     const DEFAULT_HTML_NAME = 'results.html';
     const DEFAULT_JSON_NAME = 'results.json';
+    const DEFAULT_JUNIT_NAME = 'results.xml';
     const DEFAULT_MARKDOWN_NAME = 'results.md';
     const DEFAULT_PDF_NAME = 'results.pdf';
 
@@ -184,6 +185,16 @@ function parseArgs(options, raw_args) {
         defaultValue: DEFAULT_JSON_NAME,
         help: 'JSON file to write to. Defaults to %(defaultValue)s .',
     });
+    results_group.addArgument(['--junit'], {
+        action: 'storeTrue',
+        help: 'Write tests results as a JUnit XML file.',
+    });
+    results_group.addArgument(['--junit-file'], {
+        metavar: 'FILE.xml',
+        dest: 'junit_file',
+        defaultValue: DEFAULT_JUNIT_NAME,
+        help: 'JUnit XML file to write to. Defaults to %(defaultValue)s .',
+    });
     results_group.addArgument(['-H', '--html'], {
         action: 'storeTrue',
         help: 'Write test results as an HTML file.',

src/render.js

@@ -88,7 +88,7 @@ function craftResults(config, test_info) {
 async function doRender(config, results) {
     output.logVerbose(
         config,
-        `[results] Render results JSON: ${config.json}, Markdown: ${config.markdown}, HTML: ${config.html}, PDF: ${config.pdf}`
+        `[results] Render results JSON: ${config.json}, JUnit: ${config.junit}, Markdown: ${config.markdown}, HTML: ${config.html}, PDF: ${config.pdf}`
     );
 
     if (config.json) {
@@ -125,6 +125,14 @@ async function doRender(config, results) {
         output.logVerbose(config, `Rendering to PDF ${config.pdf_file}...`);
         await pdf(config, config.pdf_file, results);
     }
+
+    if (config.junit) {
+        output.logVerbose(config, `Rendering to JUnit ${config.junit_file}...`);
+        const junit_xml = junit(results);
+        await fs.promises.writeFile(config.junit_file, junit_xml, {
+            encoding: 'utf-8',
+        });
+    }
 }
 
 function format_duration(ms) {
@@ -650,6 +658,48 @@ async function pdf(config, path, results) {
     return html2pdf(config, path, html(results));
 }
 
+/**
+ * @param {import('./internal').CraftedResults} results
+ */
+function junit(results) {
+    const testcases = results.tests.map(testResult => {
+        const { skipped, taskResults } = testResult;
+
+        // Just take the first error if there is any, on retries it would probably be the same one
+        const error = taskResults.find(tr => tr.status === 'error');
+
+        const res =
+            `<testcase classname="${testResult.group}" name="${
+                testResult.name
+            }" time="${escape_html(_calcDuration(taskResults))}">` +
+            (skipped ? `<skipped>${testResult.skipReason}</skipped>` : '') +
+            (error
+                ? `<failure>${
+                      escape_html(error.error_stack) ||
+                      'INTERNAL ERROR: no error stack'
+                  }</failure>`
+                : '') +
+            (testResult.description
+                ? `<system-out>${escape_html(
+                      testResult.description
+                  )}</system-out>`
+                : '') +
+            `</testcase>`;
+        return res;
+    });
+
+    const result =
+        '<?xml version="1.0"?>\n' +
+        '<testsuites>\n' +
+        '<testsuite>\n' +
+        testcases.join('\n') +
+        '\n' +
+        '</testsuite>\n' +
+        '</testsuites>';
+
+    return result;
+}
+
 module.exports = {
     craftResults,
     doRender,

tests/selftest_flaky_result_repeat.js

@@ -45,13 +45,21 @@ async function run() {
         }
     }
 
-    const summary = lines.slice(0, 3).map(s => s.trim());
-    assert.deepEqual(summary, [
+    const summary = lines.map(s => s.trim());
+    const expected = [
         '3 tests passed',
         '3 failed (error[0], error[1], error[2])',
         '3 flaky (flaky[0], flaky[1], flaky[2])',
-    ]);
-    assert.match(lines[3], /3 slowest tests: (.+), (.+), (.+)/);
+    ];
+    for (const item of expected) {
+        assert(
+            summary.includes(item),
+            `Expected test summary to include: ${item}\nActual summary: ${JSON.stringify(
+                summary
+            )}`
+        );
+    }
+    assert.match(lines[lines.length - 1], /3 slowest tests: (.+), (.+), (.+)/);
 }
 
 module.exports = {