Add support for saving webdriver logs in parallel test run (#4440)

Co-authored-by: Samriddhi-35 <samriddhisharma632@gmail.com>
Co-authored-by: Priyansh Garg <priyanshgarg30@gmail.com>

Author: 3 people

lib/settings/defaults.js

@@ -228,6 +228,11 @@ module.exports = {
     // leave empty to use the test suite name when writing the webdriver server logs
     log_file_name: '',
 
+    // When running tests in parallel (test workers), each worker spawns its own webdriver process but
+    // their logs are not written to disk by default. Set this to true to retain webdriver server logs
+    // from worker processes when running in parallel mode.
+    retain_logs_in_parallel_run: false,
+
     // Time to wait (in ms) before starting to check the Webdriver server is up and running
     check_process_delay: 100,
 

lib/transport/selenium-webdriver/index.js

@@ -11,6 +11,7 @@ const BaseTransport = require('../');
 const {colors} = Logger;
 const {isErrorResponse, checkLegacyResponse, throwDecodedError, WebDriverError} = error;
 const {IosSessionErrors} = require('../errors');
+const Concurrency = require('../../runner/concurrency/index.js');
 
 let _driverService = null;
 let _driver = null;
@@ -195,6 +196,14 @@ class Transport extends BaseTransport {
   async createDriverService({options, moduleKey, reuseBrowser = false}) {
     try {
       moduleKey = this.settings.webdriver.log_file_name || moduleKey || '';
+      if (Concurrency.isWorker()) {
+        // append timestamp to the output file name to differentiate between the files
+        // created across the workers.
+        moduleKey = `${moduleKey}_${process.hrtime.bigint()}`;
+        if (this.settings.webdriver.log_file_name) {
+          this.settings.webdriver.log_file_name = moduleKey;
+        }
+      }
 
       if (!this.shouldReuseDriverService(reuseBrowser)) {
         Transport.driverService = new this.ServiceBuilder(this.settings);
@@ -217,7 +226,7 @@ class Transport extends BaseTransport {
 
 
   async getDriver({options, reuseBrowser = false}) {
-    const value  = await this.shouldReuseDriver(reuseBrowser);
+    const value = await this.shouldReuseDriver(reuseBrowser);
     if (value) {
       return Transport.driver;
     }

lib/transport/selenium-webdriver/service-builders/base-service.js

@@ -217,7 +217,9 @@ class BaseService {
   }
 
   needsSinkProcess() {
-    return !Concurrency.isWorker();
+    const {retain_logs_in_parallel_run = false} = this.settings.webdriver || {};
+
+    return !Concurrency.isWorker() || retain_logs_in_parallel_run;
   }
 
   hasSinkSupport() {

test/src/service-builders/testBaseServiceConcurrency.js

@@ -0,0 +1,98 @@
+const assert = require('assert');
+const common = require('../../common.js');
+
+describe('BaseService concurrency behaviour', function () {
+  const originalParallelEnv = process.env.__NIGHTWATCH_PARALLEL_MODE;
+
+  after(function () {
+    if (originalParallelEnv === undefined) {
+      delete process.env.__NIGHTWATCH_PARALLEL_MODE;
+    } else {
+      process.env.__NIGHTWATCH_PARALLEL_MODE = originalParallelEnv;
+    }
+  });
+
+  function createService({isWorker, retainLogsInParallelRun} = {}) {
+    // Concurrency.isWorker() reads this on each call (see lib/runner/concurrency/index.js)
+    if (isWorker) {
+      process.env.__NIGHTWATCH_PARALLEL_MODE = '1';
+    } else {
+      delete process.env.__NIGHTWATCH_PARALLEL_MODE;
+    }
+
+    const BaseService = common.require('transport/selenium-webdriver/service-builders/base-service.js');
+
+    class MockBaseService extends BaseService {
+      get requiresDriverBinary() {
+        return false;
+      }
+
+      async createSinkProcess() {
+        this.sinkCreated = true;
+      }
+
+      async createService(options) {
+        // Mock the service object that BaseService.createService() expects
+        this.service = {
+          setPort: () => { },
+          setHostname: () => { },
+          setPath: () => { }
+        };
+
+        return super.createService(options);
+      }
+    }
+
+    const settings = {
+      webdriver: {
+        log_path: 'logs',
+        ...(retainLogsInParallelRun ? {retain_logs_in_parallel_run: retainLogsInParallelRun} : {})
+      }
+    };
+
+    const service = new MockBaseService(settings);
+
+    return service;
+  }
+
+  it('creates sink process when not running as worker', async function () {
+    const service = createService({isWorker: false});
+
+    assert.strictEqual(service.needsSinkProcess(), true);
+
+    await service.createService({});
+    if (process.platform === 'win32') {
+      assert.strictEqual(service.sinkCreated, undefined);
+      assert.strictEqual(service.settings.webdriver.log_path, false);
+    } else {
+      assert.strictEqual(service.sinkCreated, true);
+      assert.strictEqual(service.settings.webdriver.log_path, 'logs');
+    }
+  });
+
+  it('does not create sink and disables log_path for workers by default', async function () {
+    const service = createService({isWorker: true});
+
+    assert.strictEqual(service.needsSinkProcess(), false);
+
+    await service.createService({});
+
+    assert.strictEqual(service.sinkCreated, undefined);
+    assert.strictEqual(service.settings.webdriver.log_path, false);
+  });
+
+  it('creates sink and keeps log_path when retain_logs_in_parallel_run is true', async function () {
+    const service = createService({isWorker: true, retainLogsInParallelRun: true});
+
+    assert.strictEqual(service.needsSinkProcess(), true);
+
+    await service.createService({});
+    if (process.platform === 'win32') {
+      assert.strictEqual(service.sinkCreated, undefined);
+      assert.strictEqual(service.settings.webdriver.log_path, false);
+    } else {
+      assert.strictEqual(service.sinkCreated, true);
+      assert.strictEqual(service.settings.webdriver.log_path, 'logs');
+    }
+  });
+});

test/src/service-builders/testSeleniumServer.js

@@ -216,7 +216,8 @@ describe('SeleniumServer Transport Tests', function () {
       session,
       serverPath,
       serverPort,
-      options
+      options,
+      client
     };
   }
 
@@ -260,6 +261,126 @@ describe('SeleniumServer Transport Tests', function () {
     assert.ok(logFilePath.endsWith('testModuleKey_selenium-server.log'));
   });
 
+  it('test per-worker log file path when running in worker without log_file_name set', async function () {
+    mockery.registerMock('../../runner/concurrency/index.js', {
+      isWorker: function () {
+        return true;
+      }
+    });
+
+    mockery.registerMock('geckodriver', {
+      path: ''
+    });
+
+    mockery.registerMock('chromedriver', {
+      path: ''
+    });
+
+    mockery.registerMock('@nightwatch/selenium-server', {
+      path: '/path/to/selenium-server-standalone.3.0.jar'
+    });
+
+    let logFilePath;
+    const {client} = await SeleniumServerTestSetup({
+      desiredCapabilities: {
+        browserName: 'chrome'
+      },
+      selenium: {
+        port: 9999,
+        start_process: true
+      }
+    }, {
+      onLogFile(filePath) {
+        logFilePath = filePath;
+      }
+    });
+
+    // The log file name should include the moduleKey, timestamp and end with _selenium-server.log
+    assert.ok(/testModuleKey_[0-9]+_selenium-server\.log$/.test(logFilePath));
+    // The log_file_name should be unchanged
+    assert.strictEqual(client.settings.webdriver.log_file_name, '');
+  });
+
+  it('test log file path when not running in worker with log_file_name set', async function () {
+    mockery.registerMock('geckodriver', {
+      path: ''
+    });
+
+    mockery.registerMock('chromedriver', {
+      path: ''
+    });
+
+    mockery.registerMock('@nightwatch/selenium-server', {
+      path: '/path/to/selenium-server-standalone.3.0.jar'
+    });
+
+    let logFilePath;
+    const {client} = await SeleniumServerTestSetup({
+      desiredCapabilities: {
+        browserName: 'chrome'
+      },
+      selenium: {
+        port: 9999,
+        start_process: true
+      },
+      webdriver: {
+        log_file_name: 'customModuleKey'
+      }
+    }, {
+      onLogFile(filePath) {
+        logFilePath = filePath;
+      }
+    });
+
+    // The log file should include the customModuleKey without the timestamp.
+    assert.ok(logFilePath.endsWith('customModuleKey_selenium-server.log'));
+    // No change in the log_file_name value as we're not running in worker
+    assert.strictEqual(client.settings.webdriver.log_file_name, 'customModuleKey');
+  });
+
+  it('test per-worker log file path when running in worker with log_file_name set', async function () {
+    mockery.registerMock('../../runner/concurrency/index.js', {
+      isWorker: function () {
+        return true;
+      }
+    });
+
+    mockery.registerMock('geckodriver', {
+      path: ''
+    });
+
+    mockery.registerMock('chromedriver', {
+      path: ''
+    });
+
+    mockery.registerMock('@nightwatch/selenium-server', {
+      path: '/path/to/selenium-server-standalone.3.0.jar'
+    });
+
+    let logFilePath;
+    const {client} = await SeleniumServerTestSetup({
+      desiredCapabilities: {
+        browserName: 'chrome'
+      },
+      selenium: {
+        port: 9999,
+        start_process: true
+      },
+      webdriver: {
+        log_file_name: 'customModuleKey'
+      }
+    }, {
+      onLogFile(filePath) {
+        logFilePath = filePath;
+      }
+    });
+
+    // The log file name should include the customModuleKey, timestamp and end with _selenium-server.log
+    assert.ok(/customModuleKey_[0-9]+_selenium-server\.log$/.test(logFilePath));
+    // The log_file_name should also be updated to include the timestamp when running in worker
+    assert.ok(/^customModuleKey_[0-9]+$/.test(client.settings.webdriver.log_file_name));
+  });
+
   it('test create session with selenium server 3 -- with drivers', async function() {
     mockery.registerMock('geckodriver', {
       path: '/path/to/geckodriver'

types/nightwatch-options.d.ts

@@ -573,6 +573,15 @@ export interface WebdriverOptions {
 	 */
 	log_file_name?: string;
 
+	/**
+	 * When running tests in parallel (test workers), each worker spawns its own webdriver process but
+	 * their logs are not written to disk by default. Set this to true to retain webdriver server logs
+	 * from worker processes when running in parallel mode.
+	 *
+	 * @default false
+	 */
+	retain_logs_in_parallel_run?: boolean;
+
 	/**
 	 * List of cli arguments to be passed to the Webdriver process. This varies for each Webdriver implementation.
 	 *