feat: build kite bare minimum cli reporter (#872)

* bare minimun cli reporter

* slight modification

* update snap

* PR feedback

* more pr feedback

__tests__/reporters/__snapshots__/buildkite-cli.test.ts.snap

@@ -0,0 +1,43 @@
+// Jest Snapshot v1, https://goo.gl/fbAQLP
+
+exports[`buildkite cli reporter render hook errors without steps 1`] = `
+"
+--- Journey: j1
+   
+   thrown: {
+     name: 'Error',
+     message: 'before hook failed',
+     stack: 'Error: before hook failed'
+   }
+
+✖ Took (1 seconds)
+No tests found! (0 ms) 
+
+"
+`;
+
+exports[`buildkite cli reporter writes each step to the FD 1`] = `
+"
+--- Journey: j1
+   ✖  Step: 's1' failed (1000 ms)
+   
+   thrown: { name: 'Error', message: 'step failed', stack: 'Error: step failed' }
+
+✖ Took (1 seconds)
+
+ 1 failed (0 seconds) 
+
+"
+`;
+
+exports[`buildkite cli reporter writes multiple steps to the FD 1`] = `
+"
+--- Journey: j1
+   ✓  Step: 's1' succeeded (1000 ms)
+   ✓  Step: 's1' succeeded (4000 ms)
+✓ Took (2 seconds)
+
+ 2 passed (0 seconds) 
+
+"
+`;

__tests__/reporters/buildkite-cli.test.ts

@@ -0,0 +1,146 @@
+/**
+ * MIT License
+ *
+ * Copyright (c) 2020-present, Elastic NV
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ *
+ */
+
+/**
+ * Disable the ANSI codes for kleur/colors module
+ */
+process.env.NO_COLOR = '1';
+
+import * as fs from 'fs';
+import SonicBoom from 'sonic-boom';
+import { step, journey } from '../../src/core';
+import BuildKiteCLIReporter from '../../src/reporters/build_kite_cli';
+import * as helpers from '../../src/helpers';
+
+describe('buildkite cli reporter', () => {
+  let dest: string;
+  let stream: SonicBoom;
+  let reporter: BuildKiteCLIReporter;
+  const timestamp = 1600300800000000;
+  const j1 = journey('j1', () => {});
+
+  const readAndCloseStream = async () => {
+    /**
+     * Close the underlying stream writing to FD to read all the contents
+     */
+    stream.end();
+    await new Promise(resolve => stream.once('finish', resolve));
+    return fs.readFileSync(fs.openSync(dest, 'r'), 'utf-8');
+  };
+
+  beforeEach(() => {
+    dest = helpers.generateTempPath();
+    reporter = new BuildKiteCLIReporter({ fd: fs.openSync(dest, 'w') });
+    stream = reporter.stream;
+    jest.spyOn(helpers, 'now').mockImplementation(() => 0);
+  });
+
+  afterEach(() => {
+    fs.unlinkSync(dest);
+  });
+
+  afterAll(() => {
+    process.env.NO_COLOR = '';
+  });
+
+  it('writes each step to the FD', async () => {
+    reporter.onJourneyStart(j1, {
+      timestamp,
+    });
+    reporter.onStepEnd(j1, step('s1', helpers.noop), {
+      status: 'failed',
+      error: {
+        name: 'Error',
+        message: 'step failed',
+        stack: 'Error: step failed',
+      },
+      url: 'dummy',
+      start: 0,
+      end: 1,
+    });
+    reporter.onJourneyEnd(j1, {
+      timestamp,
+      status: 'failed',
+      error: {
+        name: 'Error',
+        message: 'step failed',
+        stack: 'Error: step failed',
+      },
+      start: 0,
+      end: 1,
+      options: {},
+    });
+    reporter.onEnd();
+    expect((await readAndCloseStream()).toString()).toMatchSnapshot();
+  });
+
+  it('render hook errors without steps', async () => {
+    reporter.onJourneyStart(j1, {
+      timestamp,
+    });
+    const error = {
+      name: 'Error',
+      message: 'before hook failed',
+      stack: 'Error: before hook failed',
+    };
+    reporter.onJourneyEnd(j1, {
+      timestamp,
+      status: 'failed',
+      error,
+      start: 0,
+      end: 1,
+      options: {},
+    });
+    reporter.onEnd();
+    expect((await readAndCloseStream()).toString()).toMatchSnapshot();
+  });
+
+  it('writes multiple steps to the FD', async () => {
+    reporter.onJourneyStart(j1, {
+      timestamp,
+    });
+    reporter.onStepEnd(j1, step('s1', helpers.noop), {
+      status: 'succeeded',
+      url: 'dummy',
+      start: 0,
+      end: 1,
+    });
+    reporter.onStepEnd(j1, step('s1', helpers.noop), {
+      status: 'succeeded',
+      url: 'http://localhost:8080',
+      start: 2,
+      end: 6,
+    });
+    reporter.onJourneyEnd(j1, {
+      timestamp,
+      status: 'succeeded',
+      start: 4,
+      end: 6,
+      options: {},
+    });
+    reporter.onEnd();
+    expect((await readAndCloseStream()).toString()).toMatchSnapshot();
+  });
+});

src/index.ts

@@ -63,7 +63,17 @@ export type {
 export type { default as Runner } from './core/runner';
 export type { Reporter, ReporterOptions } from './reporters';
 
-export type { SyntheticsConfig } from './common_types';
+export type {
+  SyntheticsConfig,
+  StepEndResult,
+  JourneyEndResult,
+  JourneyStartResult,
+  JourneyResult,
+  StepResult,
+  StartEvent,
+} from './common_types';
+
+export type { Journey, JourneyCallback } from './dsl';
 
 export type {
   Action,

src/reporters/base.ts

@@ -35,7 +35,7 @@ import {
 } from '../common_types';
 import { Journey, Step } from '../dsl';
 
-function renderDuration(durationMs) {
+export function renderDuration(durationMs) {
   return parseInt(durationMs);
 }
 

src/reporters/build_kite_cli.ts

@@ -0,0 +1,111 @@
+/**
+ * MIT License
+ *
+ * Copyright (c) 2020-present, Elastic NV
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ *
+ */
+
+import { red, green, cyan } from 'kleur/colors';
+import { ReporterOptions } from '.';
+
+import BaseReporter, { renderDuration } from './base';
+import { indent, now, symbols } from '../helpers';
+import {
+  JourneyEndResult,
+  JourneyStartResult,
+  StepEndResult,
+} from '../common_types';
+import { Journey, Step } from '../dsl';
+import { renderError, serializeError } from './reporter-util';
+
+export default class BuildKiteCLIReporter extends BaseReporter {
+  journeys: Map<string, Array<StepEndResult & { name: string }>> = new Map();
+
+  constructor(options: ReporterOptions = {}) {
+    super(options);
+  }
+
+  override onJourneyStart(journey: Journey, {}: JourneyStartResult) {
+    this.write(`\n--- Journey: ${journey.name}`);
+  }
+
+  override onStepEnd(journey: Journey, step: Step, result: StepEndResult) {
+    super.onStepEnd(journey, step, result);
+    if (!this.journeys.has(journey.name)) {
+      this.journeys.set(journey.name, []);
+    }
+    this.journeys.get(journey.name)?.push({ name: step.name, ...result });
+  }
+
+  override async onJourneyEnd(journey: Journey, endResult: JourneyEndResult) {
+    const { start, end, status } = endResult;
+    super.onJourneyEnd(journey, endResult);
+    const message = `${symbols[status]} Took (${renderDuration(
+      end - start
+    )} seconds)`;
+    this.write(message);
+  }
+
+  override onEnd() {
+    const { failed, succeeded, skipped } = this.metrics;
+    const total = failed + succeeded + skipped;
+
+    if (total > 0) {
+      const failedJourneys = Array.from(this.journeys.entries()).filter(
+        ([, steps]) => steps.some(step => step.status === 'failed')
+      );
+
+      // if more than 5 journeys, we also report failed journeys at the end
+      if (failedJourneys.length > 0 && this.journeys.size > 5) {
+        failedJourneys.forEach(([journeyName, steps]) => {
+          const name = red(`Journey: ${journeyName} :slightly_frowning_face:`);
+          this.write(`\n+++ ${name}`);
+          steps.forEach(stepResult => {
+            const { status, end, start, error, name: stepName } = stepResult;
+            const message = `${
+              symbols[status]
+            }  Step: '${stepName}' ${status} (${renderDuration(
+              (end - start) * 1000
+            )} ms)`;
+            this.write(indent(message));
+            if (error) {
+              this.write(renderError(serializeError(error)));
+            }
+          });
+        });
+      }
+    }
+
+    let message = '\n';
+    if (total === 0) {
+      message = 'No tests found!';
+      message += ` (${renderDuration(now())} ms) \n`;
+      this.write(message);
+      return;
+    }
+
+    message += succeeded > 0 ? green(` ${succeeded} passed`) : '';
+    message += failed > 0 ? red(` ${failed} failed`) : '';
+    message += skipped > 0 ? cyan(` ${skipped} skipped`) : '';
+    message += ` (${renderDuration(now() / 1000)} seconds) \n`;
+    this.write(message);
+  }
+}

src/reporters/index.ts

@@ -33,20 +33,26 @@ import {
   JourneyStartResult,
   StepEndResult,
 } from '../common_types';
+import BuildKiteCLIReporter from './build_kite_cli';
 
 export type ReporterOptions = {
   fd?: number;
   colors?: boolean;
   dryRun?: boolean;
 };
-export type BuiltInReporterName = 'default' | 'json' | 'junit';
+export type BuiltInReporterName =
+  | 'default'
+  | 'json'
+  | 'junit'
+  | 'buildkite-cli';
 export type ReporterInstance = new (opts: ReporterOptions) => Reporter;
 export const reporters: {
   [key in BuiltInReporterName]: ReporterInstance;
 } = {
   default: BaseReporter,
   json: JSONReporter,
   junit: JUnitReporter,
+  'buildkite-cli': BuildKiteCLIReporter,
 };
 
 export interface Reporter {