implement repeats

Refs: Re-run a test multiple times mochajs#2332

docs/index.md

@@ -687,6 +687,31 @@ describe('retries', function () {
 });
 ```
 
+## Repeat Tests
+
+Tests can also be repeated when they pass. This feature can be used to test for leaks and proper tear-down procedures. In this case a test is considered to be successful only if all the runs are successful.
+
+This feature does re-run a passed test and its corresponding `beforeEach/afterEach` hooks, but not `before/after` hooks.
+
+If using both `repeat` and `retries`, the test will be run `repeat` times tolerating up to `retries` failures in total.
+
+```js
+describe('repeat', function () {
+  // Repeat all tests in this suite 4 times
+  this.repeats(4);
+
+  beforeEach(function () {
+    browser.get('http://www.yahoo.com');
+  });
+
+  it('should use proper tear-down', function () {
+    // Specify this test to only retry up to 2 times
+    this.repeats(2);
+    expect($('.foo').isDisplayed()).to.eventually.be.true;
+  });
+});
+```
+
 ## Dynamically Generating Tests
 
 Given Mocha's use of function expressions to define suites and test cases, it's straightforward to generate your tests dynamically. No special syntax is required — plain ol' JavaScript can be used to achieve functionality similar to "parameterized" tests, which you may have seen in other frameworks.
@@ -1777,7 +1802,7 @@ describe('Array', function () {
       it('should not throw an error', function () {
         (function () {
           [1, 2, 3].indexOf(4);
-        }.should.not.throw());
+        }).should.not.throw();
       });
       it('should return -1', function () {
         [1, 2, 3].indexOf(4).should.equal(-1);
@@ -2152,6 +2177,7 @@ mocha.setup({
   forbidPending: true,
   global: ['MyLib'],
   retries: 3,
+  repeats: 1,
   rootHooks: { beforeEach(done) { ... done();} },
   slow: '100',
   timeout: '2000',

example/config/.mocharc.js

@@ -36,6 +36,7 @@ module.exports = {
   'reporter-option': ['foo=bar', 'baz=quux'], // array, not object
   require: '@babel/register',
   retries: 1,
+  repeats: 1,
   slow: '75',
   sort: false,
   spec: ['test/**/*.spec.js'], // the positional arguments!

example/config/.mocharc.yml

@@ -40,6 +40,7 @@ reporter-option: # array, not object
   - 'baz=quux'
 require: '@babel/register'
 retries: 1
+repeats: 1
 slow: '75'
 sort: false
 spec:

lib/cli/run-option-metadata.js

@@ -49,7 +49,7 @@ const TYPES = (exports.types = {
     'sort',
     'watch'
   ],
-  number: ['retries', 'jobs'],
+  number: ['retries', 'repeats', 'jobs'],
   string: [
     'config',
     'fgrep',

lib/cli/run.js

@@ -231,6 +231,10 @@ exports.builder = yargs =>
         description: 'Retry failed tests this many times',
         group: GROUPS.RULES
       },
+      repeats: {
+        description: 'Repeat passed tests this many times',
+        group: GROUPS.RULES
+      },
       slow: {
         default: defaults.slow,
         description: 'Specify "slow" test threshold (in milliseconds)',

lib/context.js

@@ -84,3 +84,18 @@ Context.prototype.retries = function (n) {
   this.runnable().retries(n);
   return this;
 };
+
+/**
+ * Set or get a number of repeats on passed tests
+ *
+ * @private
+ * @param {number} n
+ * @return {Context} self
+ */
+Context.prototype.repeats = function (n) {
+  if (!arguments.length) {
+    return this.runnable().repeats();
+  }
+  this.runnable().repeats(n);
+  return this;
+};

lib/hook.js

@@ -63,6 +63,7 @@ Hook.prototype.error = function (err) {
 Hook.prototype.serialize = function serialize() {
   return {
     $$currentRetry: this.currentRetry(),
+    $$currentRepeat: this.currentRepeat(),
     $$fullTitle: this.fullTitle(),
     $$isPending: Boolean(this.isPending()),
     $$titlePath: this.titlePath(),

lib/mocha.js

@@ -170,6 +170,7 @@ exports.run = function (...args) {
  * @param {string|constructor} [options.reporter] - Reporter name or constructor.
  * @param {Object} [options.reporterOption] - Reporter settings object.
  * @param {number} [options.retries] - Number of times to retry failed tests.
+ * @param {number} [options.repeat] - Number of times to repeat passed tests.
  * @param {number} [options.slow] - Slow threshold value.
  * @param {number|string} [options.timeout] - Timeout threshold value.
  * @param {string} [options.ui] - Interface name.
@@ -207,6 +208,10 @@ function Mocha(options = {}) {
     this.retries(options.retries);
   }
 
+  if ('repeats' in options) {
+    this.repeats(options.repeats);
+  }
+
   [
     'allowUncaught',
     'asyncOnly',
@@ -763,6 +768,25 @@ Mocha.prototype.retries = function (retry) {
   return this;
 };
 
+/**
+ * Sets the number of times to repeat passed tests.
+ *
+ * @public
+ * @see [CLI option](../#-repeats-n)
+ * @see [Repeat Tests](../#repeat-tests)
+ * @param {number} repeats - Number of times to repeat passed tests.
+ * @return {Mocha} this
+ * @chainable
+ * @example
+ *
+ * // Allow any passed test to be repeated multiple times
+ * mocha.repeats(1);
+ */
+Mocha.prototype.repeats = function (repeats) {
+  this.suite.repeats(repeats);
+  return this;
+};
+
 /**
  * Sets slowness threshold value.
  *

lib/reporters/json-stream.js

@@ -85,6 +85,7 @@ function clean(test) {
     file: test.file,
     duration: test.duration,
     currentRetry: test.currentRetry(),
+    currentRepeat: test.currentRepeat(),
     speed: test.speed
   };
 }

lib/reporters/json.js

@@ -115,6 +115,7 @@ function clean(test) {
     file: test.file,
     duration: test.duration,
     currentRetry: test.currentRetry(),
+    currentRepeat: test.currentRepeat(),
     speed: test.speed,
     err: cleanCycles(err)
   };

lib/runnable.js

@@ -40,6 +40,7 @@ function Runnable(title, fn) {
   this._timeout = 2000;
   this._slow = 75;
   this._retries = -1;
+  this._repeats = 1;
   utils.assignNewMochaID(this);
   Object.defineProperty(this, 'id', {
     get() {
@@ -60,6 +61,7 @@ utils.inherits(Runnable, EventEmitter);
 Runnable.prototype.reset = function () {
   this.timedOut = false;
   this._currentRetry = 0;
+  this._currentRepeat = 1;
   this.pending = false;
   delete this.state;
   delete this.err;
@@ -182,6 +184,18 @@ Runnable.prototype.retries = function (n) {
   this._retries = n;
 };
 
+/**
+ * Set or get number of repeats.
+ *
+ * @private
+ */
+Runnable.prototype.repeats = function (n) {
+  if (!arguments.length) {
+    return this._repeats;
+  }
+  this._repeats = n;
+};
+
 /**
  * Set or get current retry
  *
@@ -194,6 +208,18 @@ Runnable.prototype.currentRetry = function (n) {
   this._currentRetry = n;
 };
 
+/**
+ * Set or get current repeat
+ *
+ * @private
+ */
+Runnable.prototype.currentRepeat = function (n) {
+  if (!arguments.length) {
+    return this._currentRepeat;
+  }
+  this._currentRepeat = n;
+};
+
 /**
  * Return the full title generated by recursively concatenating the parent's
  * full title.

lib/runner.js

@@ -814,6 +814,14 @@ Runner.prototype.runTests = function (suite, fn) {
             self.fail(test, err);
           }
           self.emit(constants.EVENT_TEST_END, test);
+          return self.hookUp(HOOK_TYPE_AFTER_EACH, next);
+        } else if (test.currentRepeat() < test.repeats()) {
+          var repeatedTest = test.clone();
+          repeatedTest.currentRepeat(test.currentRepeat() + 1);
+          tests.unshift(repeatedTest);
+
+          self.emit(constants.EVENT_TEST_RETRY, test, null);
+
           return self.hookUp(HOOK_TYPE_AFTER_EACH, next);
         }
 

lib/suite.js

@@ -73,6 +73,7 @@ function Suite(title, parentContext, isRoot) {
   this.root = isRoot === true;
   this.pending = false;
   this._retries = -1;
+  this._repeats = 1;
   this._beforeEach = [];
   this._beforeAll = [];
   this._afterEach = [];
@@ -127,6 +128,7 @@ Suite.prototype.clone = function () {
   suite.root = this.root;
   suite.timeout(this.timeout());
   suite.retries(this.retries());
+  suite.repeats(this.repeats());
   suite.slow(this.slow());
   suite.bail(this.bail());
   return suite;
@@ -174,6 +176,22 @@ Suite.prototype.retries = function (n) {
   return this;
 };
 
+/**
+ * Set or get number of times to repeat a passed test.
+ *
+ * @private
+ * @param {number|string} n
+ * @return {Suite|number} for chaining
+ */
+Suite.prototype.repeats = function (n) {
+  if (!arguments.length) {
+    return this._repeats;
+  }
+  debug('repeats %d', n);
+  this._repeats = parseInt(n, 10) || 0;
+  return this;
+};
+
 /**
  * Set or get slow `ms` or short-hand such as "2s".
  *
@@ -230,6 +248,7 @@ Suite.prototype._createHook = function (title, fn) {
   hook.parent = this;
   hook.timeout(this.timeout());
   hook.retries(this.retries());
+  hook.repeats(this.repeats());
   hook.slow(this.slow());
   hook.ctx = this.ctx;
   hook.file = this.file;
@@ -344,6 +363,7 @@ Suite.prototype.addSuite = function (suite) {
   suite.root = false;
   suite.timeout(this.timeout());
   suite.retries(this.retries());
+  suite.repeats(this.repeats());
   suite.slow(this.slow());
   suite.bail(this.bail());
   this.suites.push(suite);
@@ -362,6 +382,7 @@ Suite.prototype.addTest = function (test) {
   test.parent = this;
   test.timeout(this.timeout());
   test.retries(this.retries());
+  test.repeats(this.repeats());
   test.slow(this.slow());
   test.ctx = this.ctx;
   this.tests.push(test);

lib/test.js

@@ -73,7 +73,9 @@ Test.prototype.clone = function () {
   test.timeout(this.timeout());
   test.slow(this.slow());
   test.retries(this.retries());
+  test.repeats(this.repeats());
   test.currentRetry(this.currentRetry());
+  test.currentRepeat(this.currentRepeat());
   test.retriedTest(this.retriedTest() || this);
   test.globals(this.globals());
   test.parent = this.parent;
@@ -91,6 +93,7 @@ Test.prototype.clone = function () {
 Test.prototype.serialize = function serialize() {
   return {
     $$currentRetry: this._currentRetry,
+    $$currentRepeat: this._currentRepeat,
     $$fullTitle: this.fullTitle(),
     $$isPending: Boolean(this.pending),
     $$retriedTest: this._retriedTest || null,

test/assertions.js

@@ -306,6 +306,24 @@ module.exports = {
           });
         }
       )
+      .addAssertion(
+        '<JSONResult> [not] to have repeated test <string>',
+        (expect, result, title) => {
+          expect(result.tests, '[not] to have an item satisfying', {
+            title,
+            currentRepeat: expect.it('to be positive')
+          });
+        }
+      )
+      .addAssertion(
+        '<JSONResult> [not] to have repeated test <string> <number>',
+        (expect, result, title, count) => {
+          expect(result.tests, '[not] to have an item satisfying', {
+            title,
+            currentRepeat: count
+          });
+        }
+      )
       .addAssertion(
         '<JSONResult> [not] to have failed with (error|errors) <any+>',
         function (expect, result, ...errors) {

test/integration/events.spec.js

@@ -58,6 +58,25 @@ describe('event order', function () {
     });
   });
 
+  describe('--repeats test case', function () {
+    it('should assert --repeats event order', function (done) {
+      runMochaJSON(
+        'runner/events-repeats.fixture.js',
+        ['--repeats', '2'],
+        function (err, res) {
+          if (err) {
+            done(err);
+            return;
+          }
+          expect(res, 'to have passed')
+            .and('to have failed test count', 0)
+            .and('to have passed test count', 1);
+          done();
+        }
+      );
+    });
+  });
+
   describe('--delay test case', function () {
     it('should assert --delay event order', function (done) {
       runMochaJSON(

test/integration/fixtures/options/parallel/repeats.fixture.js

@@ -0,0 +1,14 @@
+describe('repeats suite', function() {
+  let calls = 0;
+  this.repeats(3);
+
+  it('should pass', function() {
+
+  });
+
+  it('should fail on the second call', function () {
+    calls++;
+    console.log(`RUN: ${calls}`);
+    if (calls > 1) throw new Error();
+  });
+});

test/integration/fixtures/options/repeats.fixture.js

@@ -0,0 +1,5 @@
+'use strict';
+
+describe('repeats', function () {
+  it('should pass', () => undefined);
+});