multicapabilities not showing other reports when more capabilities in array

[lordgreg]

When using multiCapabilities, having more than one item in array and using jasmine-spec-reporter, the dots still show up in between each of tests made. However, this is not the case for jasmine-reporter, which kicks in only, after the full capability (spec/suite) has been done.

This, it looks like, is not reporter issue (issue created on jasmine-spec-reporter github) but how protractor handles the specs/reporters.

Please see where the dots are being reported and where after them the jasmine-reporter reports are.

Bug report

• Node Version: 7.7.1
• Protractor Version: 4.0.14
• Angular Version: 1.5.3
• Browser(s): Chrome
• Operating System and Version MacOS Sierra 10.12.3 (16D32)
• Your protractor configuration file

'use strict';

var SpecReporter = require('jasmine-spec-reporter').SpecReporter;

// An example configuration file.
exports.config = {
  // The address of a running selenium server.
  //seleniumAddress: 'http://localhost:4444/wd/hub',
  //seleniumServerJar: deprecated, this should be set on node_modules/protractor/config.json

  // Capabilities to be passed to the webdriver instance.
  maxSessions: 1,
  multiCapabilities: [
    {
      browserName: 'chrome',
      chromeOptions: {
        'binary': '/Applications/Google\ Chrome\ Canary.app/Contents/MacOS/Google\ Chrome\ Canary',
        'args': [
          '--disable-web-security',
          '--window-size=768,1024'
        ]
      }
    },
    {
      browserName: 'chrome',
      chromeOptions: {
        'binary': '/Applications/Google\ Chrome\ Canary.app/Contents/MacOS/Google\ Chrome\ Canary',
        'args': [
          '--disable-web-security',
          '--window-size=375,667'
        ]
      }
    },
  ],

  baseUrl: 'http://localhost:3000/',

  // Spec patterns are relative to the current working directly when
  // protractor is called.
  specs: ['test/protractor/**/*.js'],

  suites: {
    login: 'test/protractor/login/**/*.spec.js',
    catalogs: 'test/protractor/catalogs/**/*.spec.js'
  },

  jasmineNodeOpts: {
    showColors: true,
    silent: true,
    // defaultTimeoutInterval: 360000,
    print: function () {
    }
  },

  onPrepare: function () {
    /*globals jasmine*/
    jasmine.getEnv().addReporter(new SpecReporter({
      spec: {
        displayStacktrace: true
      }
    }));
  }

};

• A relevant example test

login.model.js

'use strict';

var LoginPage = function () {
  this.email = element(by.model('LoginCtrl.data.formData.email'));
  this.password = element(by.model('LoginCtrl.data.formData.password'));
  this.submit = element(by.css('.login form button'));
  this.errorPopup = element(by.css('.popup-container .popup'));

  this.setEmail = function (param) {
    this.email.sendKeys(param);
  };

  this.setPassword = function (param) {
    this.password.sendKeys(param);
  };

  this.resetForm = function () {
    this.email.clear();
    this.password.clear();
  };

  this.submitForm = function () {
    this.submit.click();
  };
};

module.exports.LoginPage = LoginPage;

login.spec.js:

'use strict';

var LoginModel = require('./login.model.js');
var loginPage;

describe('Login', function () {

  beforeEach(function () {
    loginPage = new LoginModel.LoginPage();
    browser.get('/#/app/login');
  });

  it('Should login as user AAA', function (done) {
    loginPage.email.clear();
    loginPage.setEmail('aaa@aaa.de');
    loginPage.setPassword('aaa');
    loginPage.submit.click();

    browser.wait(function () {
      return browser.driver.getCurrentUrl().then(function (url) {
        return !/login/.test(url);
      });
    }, 8000)
    .then(function () {
      browser.getCurrentUrl()
      .then(function (url) {
        expect(url).not.toContain('app/login');
        done();
      });

    });

  });

  it('Should fail the login as user banana', function () {
    loginPage.email.clear();
    loginPage.setEmail('banana@fruit.de');
    loginPage.setPassword('aaaaaa');
    loginPage.submit.click();

    expect(browser.getCurrentUrl()).toContain('app/login');
    expect(loginPage.errorPopup.isPresent()).toBeTruthy();
  });

  it('Should have disabled login button if no email or empty fields', function () {
    loginPage.resetForm();
    expect(loginPage.submit.isEnabled()).toBeFalsy();

    loginPage.resetForm();
    loginPage.setEmail('aaaa');
    loginPage.setPassword('aaaa');
    expect(loginPage.submit.isEnabled()).toBeFalsy();

    loginPage.resetForm();
    loginPage.setEmail('aaaa');
    loginPage.setPassword('');
    expect(loginPage.submit.isEnabled()).toBeFalsy();

    loginPage.resetForm();
    loginPage.setEmail('');
    loginPage.setPassword('aaaa');
    expect(loginPage.submit.isEnabled()).toBeFalsy();

    loginPage.resetForm();
    loginPage.setEmail('test@test.com');
    loginPage.setPassword('aaaa');
    expect(loginPage.submit.isEnabled()).toBeTruthy();
  });

});

• Output from running the test

npm run test:protractor -- --suite login   

> @ test:protractor /Users/SSSSSS/Projects/XXX
> protractor protractor.conf.js "--suite" "login"

(node:28933) DeprecationWarning: os.tmpDir() is deprecated. Use os.tmpdir() instead.
[07:49:23] I/launcher - Running 1 instances of WebDriver
...[07:49:53] I/testLogger - 
------------------------------------

[07:49:53] I/testLogger - [chrome #01] PID: 28934
[chrome #01] Specs: /Users/SSSSSS/Projects/XXX/test/protractor/login/login.spec.js
[chrome #01] 
[chrome #01] (node:28934) DeprecationWarning: os.tmpDir() is deprecated. Use os.tmpdir() instead.
[chrome #01] [07:49:23] I/local - Starting selenium standalone server...
[chrome #01] [07:49:24] I/local - Selenium standalone server started at http://10.21.4.167:64285/wd/hub
[chrome #01] Spec started
[chrome #01] 
[chrome #01]   Login
[chrome #01]     ✓ Should login as user AAA
[chrome #01]     ✓ Should fail the login as user banana
[chrome #01]     ✓ Should have disabled login button if no email or empty fields
[chrome #01] 
[chrome #01] Executed 3 of 3 specs SUCCESS in 28 secs.
[chrome #01] [07:49:53] I/local - Shutting down selenium standalone server.

[07:49:53] I/testLogger - 

[07:49:53] I/launcher - 1 instance(s) of WebDriver still running
...[07:50:26] I/testLogger - 
------------------------------------

[07:50:26] I/testLogger - [chrome #11] PID: 28956
[chrome #11] Specs: /Users/SSSSSS/Projects/XXX/test/protractor/login/login.spec.js
[chrome #11] 
[chrome #11] (node:28956) DeprecationWarning: os.tmpDir() is deprecated. Use os.tmpdir() instead.
[chrome #11] [07:49:54] I/local - Starting selenium standalone server...
[chrome #11] [07:49:54] I/local - Selenium standalone server started at http://10.21.4.167:62207/wd/hub
[chrome #11] Spec started
[chrome #11] 
[chrome #11]   Login
[chrome #11]     ✓ Should login as user AAA
[chrome #11]     ✓ Should fail the login as user banana
[chrome #11]     ✓ Should have disabled login button if no email or empty fields
[chrome #11] 
[chrome #11] Executed 3 of 3 specs SUCCESS in 31 secs.
[chrome #11] [07:50:26] I/local - Shutting down selenium standalone server.

[07:50:26] I/testLogger - 

[07:50:26] I/launcher - 0 instance(s) of WebDriver still running
[07:50:26] I/launcher - chrome #01 passed
[07:50:26] I/launcher - chrome #11 passed

• Steps to reproduce the bug
  Just use the jasmine-spec-reporter (npm install --save-dev jasmine-spec-reporter and config above)
• The URL you are running your tests against (if relevant)
  localhost:3000

[NickTomlin]

Hmm, thanks for bringing this up. Could I get you to create a small repository that reproduces it? We can use that to work towards a potential fix.

The expected output for you would be that the jasmine reporter output would be the only output? Or is it that it would print out in "real time" alongside the dots?

Thanks!

[lordgreg]

Hi @NickTomlin,

I've created a Project which can be found here. After cloning,

> npm i && bower i
> npm run protractor

Tests will run through two times, jasmine reports will shown only after session is completely done. If you need any additional help about project, please let me know.

My expected output would be to only see Jasmine and the reporter should also show the result after EACH test, not after each session (that means it should work exactly as it does if ONLY ONE capability is used)

[NickTomlin]

Awesome, thanks for doing that. We are a little behind in the queue but this is a great start for us.

[zeljkosimic95]

Hi @NickTomlin , has there been any progress on this issue ? We've made our reporter and we're having the same problem when using multicapabilities, but also the "shardTestFiles" capability, that also runs multiple browsers at the same time.

[debasispanda]

Its been open for long. Are we expecting this feature in coming days?

[viktorgogulenko]

Hey, guys. Seems this issue is a big problem for my tests - I can't run them in parallel because all kind of reporters are broken: as soon as I'm trying to use multicapabilities and 'shardTestFiles: true' TestRail reporter is no more working, Allure reporter - is no more working etc. So I can't run tests in parallel and can't receive results. I'm really thinking now to switch to some another framework just because of this problem - what's a reason to run hundreds of tests sequentially and not be able to use parallel runs? Thanks in advance for fix!