Add Soft Assertions

[zx8086]

Would be great if we could do soft assertions like in Playwright as shown in the example / link below.

https://playwright.dev/docs/test-assertions#soft-assertions

`// Make a few checks that will not stop the test when failed...
await expect.soft(page.getByTestId('status')).toHaveText('Success');
await expect.soft(page.getByTestId('eta')).toHaveText('1 day');

// ... and continue the test to check more things.
await page.getByRole('link', { name: 'next page' }).click();
await expect.soft(page.getByRole('heading', { name: 'Make another order' })).toBeVisible();`

Tasks

Beta Give feedback

No tasks being tracked yet.

[shahzad31]

@zx8086 i think we already supports those assertions from playwright, which version of synthetics are you using?

#812

[shahzad31]

Note that soft assertions only work with Playwright test runner.

i now get it, i think it needs some handling on our runner to make it work

[zx8086]

@shahzad31 This will vastly improve API testing

[vigneshshanmugam]

We added a way to do soft assertion on the Step level by marking the step with .soft. Failure of soft step will not skip rest of the steps in the journey.

journey('test journey', ({}) => {
  step.soft('step 1', async () => {
   // errored step
    expect('').toEqual('hello');
  });
    // step 2 will still run
  step('step 2', async () => {
    // errored step
    expect('').toEqual('hello');
  });
});

From version 1.7.0 - https://github.com/elastic/synthetics/releases/tag/v1.7.0

[zx8086]

@vigneshshanmugam I get the following error when I add ".soft" to a step in my journey

error executing step: command '/tmp/elastic-synthetics-unzip-2818372158/node_modules/.bin/elastic-synthetics /tmp/elastic-synthetics-unzip-2818372158/node_modules/.bin/elastic-synthetics /tmp/elastic-synthetics-unzip-2818372158 --playwright-options {"headless":true,"ignoreHTTPSErrors":false} --screenshots off --no-throttling --rich-events --match DS - API Journey | Divisions (core) - prd --params "{1 hidden params}"' exited unexpectedly with code: 1

[vigneshshanmugam]

@zx8086 Could you tell the stack/heartbeat version used for testing this? You might need to upgrade to 8.13.x if its an older version.

[zx8086]

@vigneshshanmugam Upgrade Elastic-Complete-Agent to 8.13.4 and that resolved the error but the Soft Assertions does not seem to be working ?

[vigneshshanmugam]

@zx8086 Interesting, I just did ran a fake broken test to see if I can repro this behaviour, unfortunately I cant.

  step('step 1', async () => {
    browser.newContext();
    expect(page).toHaveTitle('blah');
  });

  step.soft('step 2', async () => {
    // errored step
    expect('').toEqual('hello');
  });
  // step 3 will still run
  step('step 3', async () => {
    console.log('step 3');
  });

Can you run this test in a Public synthetics location and see if there is a difference from your private location. Also, if you can share a script that exposes this behaviour - I can double check on my side. Thanks.

[zx8086]

@vigneshshanmugam Here is the full script, I will try it with a Public synthetic location tomorrow

import { journey, step, monitor, expect } from '@elastic/synthetics';

journey('DS - API Journey | Divisions (core) - prd', async ({ context }: { context: any }) => {
  monitor.use({
    id: 'Divisions_Core',
    schedule: 30,
    screenshot: 'off',
    throttling: false,
    tags: ['production', 'digital_selling', 'eu-shared-services'],
  });

// GET all the divisions for a brand /division/brand
  step.soft('GET - Get all the Brands with corresponding Division codes', async () => {
    const brandResponse = await context.request.get('https://divisions.prd.shared-services.eu.pvh.cloud/v1/brands');
    const brandBody = await brandResponse.json();
    console.log(JSON.stringify(brandBody));

    verifyResponse(brandResponse, 200);
    verifyResponseStructure(brandBody);
    verifyCompanyDetails(brandBody);
  });

  step('GET - Get a Division for a Brand', async () => {
    const divisionResponse = await context.request.get('https://divisions.prd.shared-services.eu.pvh.cloud/v1/divisions/TH/division/01');
    const divisionBody = await divisionResponse.json();
    console.log(JSON.stringify(divisionBody));

    verifyResponse(divisionResponse, 200);
    verifyResponseStructureDivisionBrand(divisionBody);
    validateDateFields(divisionBody);
  });

  step('GET - Get a Season for a Brand', async () => {
    const seasonResponse = await context.request.get('https://divisions.prd.shared-services.eu.pvh.cloud/v1/divisions/TH/division/01/seasons/C41');
    const seasonBody = await seasonResponse.json();
    console.log(JSON.stringify(seasonBody));

    verifyResponse(seasonResponse, 200);
    verifyResponseStructureSeasonBrand(seasonBody);
    validateDateValuesSeasonBrand(seasonBody);
  });

  function verifyResponse(response, expectedStatusCode) {
    expect(response.status()).toBe(expectedStatusCode);
  }

  function verifyResponseStructure(responseBody) {
    expect(responseBody).toHaveProperty('items');
    expect(Array.isArray(responseBody.items)).toBe(true);
    expect(responseBody).toHaveProperty('count');
    expect(typeof responseBody.count).toBe('number');
    expect(responseBody).toHaveProperty('hasNextPage');
    expect(typeof responseBody.hasNextPage).toBe('boolean');
    expect(responseBody).toHaveProperty('skipToken');
  }

  function verifyCompanyDetails(responseBody) {
    const items = responseBody.items;
    expect(items).toHaveLength(7);


    // Verify details for each company code
    const companyDetails = [
      {
        companyCode: 'THEU',
        name: 'Tommy Hilfiger',
        divisionCodes: ['01','02','03','04','05','07','08','09','10','11','18'],
        salesOrganizationCodes: ['THE1', 'INLC'],
      },
      {
        companyCode: 'CKEU',
        name: 'Calvin Klein',
        divisionCodes: ['61', '62', '63', '65', '67', '68', '69', '70', '71', '72', '77'],
        salesOrganizationCodes: ['CK07', 'INCK'],
      },
      {
        companyCode: 'MK',
        name: 'Michael Kors',
        divisionCodes: ['91'],
        salesOrganizationCodes: ['PBE1'],
      },
      {
        companyCode: 'IZEU',
        name: 'IZOD',
        divisionCodes: ['92'],
        salesOrganizationCodes: ['PBE1'],
      },
      {
        companyCode: 'NIKE',
        name: 'NIKE',
        divisionCodes: ['97'],
        salesOrganizationCodes: ['PBE1'],
      },
    ];

    for (const detail of companyDetails) {
      const company = items.find((item) => item.companyCode === detail.companyCode);
      expect(company).toBeDefined();
      expect(company.name).toBe(detail.name);
      expect(company.divisionCodes).toEqual(expect.arrayContaining(detail.divisionCodes));
      expect(company.salesOrganizationCodes).toEqual(expect.arrayContaining(detail.salesOrganizationCodes));
    }
  }

  function verifyResponseStructureDivisionBrand(responseBody) {
    expect(responseBody).toHaveProperty('name');
    expect(responseBody).toHaveProperty('code');
    expect(responseBody).toHaveProperty('weCTimestamp');
    expect(responseBody).toHaveProperty('modifiedOn');
    expect(responseBody).toHaveProperty('seasons');
    expect(Array.isArray(responseBody.seasons)).toBe(true);
  }

  function validateDateFields(responseBody) {
    const dateFields = ['weCTimestamp', 'modifiedOn'];
    for (const field of dateFields) {
      const fieldValue = new Date(responseBody[field]);
      expect(fieldValue).toBeInstanceOf(Date);
      expect(!isNaN(fieldValue)).toBe(true);
    }
  }

  function verifyResponseStructureSeasonBrand(responseBody) {
    expect(responseBody).toHaveProperty('name');
    expect(responseBody).toHaveProperty('code');
    expect(responseBody).toHaveProperty('seasons');
    expect(Array.isArray(responseBody.seasons)).toBe(true);
  }

  function validateDateValuesSeasonBrand(responseBody) {
    const seasons = responseBody.seasons;
    expect(seasons).toHaveLength(1);

    const season = seasons[0];
    expect(season).toHaveProperty('name');
    expect(season).toHaveProperty('styleSeasonCode');
    expect(season).toHaveProperty('orderFulfillmentTypes');
    expect(Array.isArray(season.orderFulfillmentTypes)).toBe(true);

    const dateFields = season.orderFulfillmentTypes.map((type) => type.deliveryDates).flat();
    for (const date of dateFields) {
      const parsedDate = new Date(date);
      expect(parsedDate).toBeInstanceOf(Date);
      expect(!isNaN(parsedDate)).toBe(true);
    }
  }
});